diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a412569fbd477f02a0b67b83e8814ab98b34031..f2f1bd6263e917ff4c9c78f7d851b61cacc6c57f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -242,6 +242,7 @@ build-theodolite-native: script: - gu install native-image # TODO move to image - ./gradlew --build-cache assemble -Dquarkus.package.type=native + when: manual artifacts: paths: - "theodolite-quarkus/build/*-runner" @@ -252,7 +253,7 @@ test-theodolite: extends: .theodolite needs: - build-theodolite-jvm - - build-theodolite-native + #- build-theodolite-native script: ./gradlew test --stacktrace # Disabled for now @@ -279,12 +280,13 @@ deploy-theodolite: - .theodolite - .dind needs: - - build-theodolite-native + #- build-theodolite-native + - build-theodolite-jvm - test-theodolite script: - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - - docker build -f src/main/docker/Dockerfile.native -t theodolite . - #- docker build -f src/main/docker/Dockerfile.jvm -t theodolite . + #- docker build -f src/main/docker/Dockerfile.native -t theodolite . + - docker build -f src/main/docker/Dockerfile.jvm -t theodolite . - "[ ! $CI_COMMIT_TAG ] && docker tag theodolite $CR_HOST/$CR_ORG/theodolite:${DOCKER_TAG_NAME}latest" - "[ ! $CI_COMMIT_TAG ] && docker tag theodolite $CR_HOST/$CR_ORG/theodolite:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA" - "[ $CI_COMMIT_TAG ] && docker tag theodolite $CR_HOST/$CR_ORG/theodolite:$CI_COMMIT_TAG" @@ -305,10 +307,23 @@ deploy-theodolite: # Theodolite SLO Checker: Lag Trend +test-slo-checker-lag-trend: + stage: test + image: python:3.7-slim + tags: + - exec-docker + script: + - cd slope-evaluator + - pip install -r requirements.txt + - cd app + - python -m unittest + deploy-slo-checker-lag-trend: stage: deploy extends: - .dind + needs: + - test-slo-checker-lag-trend script: - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - docker build --pull -t theodolite-slo-checker-lag-trend slope-evaluator @@ -335,6 +350,7 @@ deploy-random-scheduler: stage: deploy extends: - .dind + needs: [] script: - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - docker build --pull -t theodolite-random-scheduler execution/infrastructure/random-scheduler diff --git a/execution/README.md b/execution/README.md index ca15111c0ad7000a200c0c50427a2c2aeb75e093..eb6ade9f2aab28b6d237e9622f22da0ea5998a50 100644 --- a/execution/README.md +++ b/execution/README.md @@ -225,7 +225,17 @@ Theodolite locally on your machine see the description below. see the [Configuration](#configuration) section below. Note, that you might uncomment the `serviceAccountName` line if RBAC is enabled on your cluster (see installation of [Theodolite RBAC](#Theodolite-RBAC)). -To start the execution of a benchmark run (with `<your-theodolite-yaml>` being your job definition): +To start the execution of a benchmark create a ConfigMap which containts all required Kubernetes resource files for the SUT and the load generator, a ConfigMap for the execution and a ConfigMap for the benchmark. + +```sh +kubectl create configmap app-resources-configmap --from-file=<folder-with-all-required-k8s-resources> +kubectl create configmap execution-configmap --from-file=<execution.yaml> +kubectl create configmap benchmark-configmap --from-file=<benchmark.yaml> +``` + +This will create three ConfigMaps. You can verify this via `kubectl get configmaps`. + +Start the Theodolite job (with `<your-theodolite-yaml>` being your job definition): ```sh kubectl create -f <your-theodolite-yaml> @@ -241,24 +251,7 @@ Kubernetes volume. ### Configuration -| Command line | Kubernetes | Description | -| -------------------- | ------------------- | ------------------------------------------------------------ | -| --uc | UC | **[Mandatory]** Stream processing use case to be benchmarked. Has to be one of `1`, `2`, `3` or `4`. | -| --loads | LOADS | **[Mandatory]** Values for the workload generator to be tested, should be sorted in ascending order. | -| --instances | INSTANCES | **[Mandatory]** Numbers of instances to be benchmarked, should be sorted in ascending order. | -| --duration | DURATION | Duration in minutes subexperiments should be executed for. *Default:* `5`. | -| --partitions | PARTITIONS | Number of partitions for Kafka topics. *Default:* `40`. | -| --cpu-limit | CPU_LIMIT | Kubernetes CPU limit for a single Pod. *Default:* `1000m`. | -| --memory-limit | MEMORY_LIMIT | Kubernetes memory limit for a single Pod. *Default:* `4Gi`. | -| --domain-restriction | DOMAIN_RESTRICTION | A flag that indiciates domain restriction should be used. *Default:* not set. For more details see Section [Domain Restriction](#domain-restriction). | -| --search-strategy | SEARCH_STRATEGY | The benchmarking search strategy. Can be set to `check-all`, `linear-search` or `binary-search`. *Default:* `check-all`. For more details see Section [Benchmarking Search Strategies](#benchmarking-search-strategies). | -| --reset | RESET | Resets the environment before each subexperiment. Useful if execution was aborted and just one experiment should be executed. | -| --reset-only | RESET_ONLY | Only resets the environment. Ignores all other parameters. Useful if execution was aborted and one want a clean state for new executions. | -| --namespace | NAMESPACE | Kubernetes namespace. *Default:* `default`. | -| --prometheus | PROMETHEUS_BASE_URL | Defines where to find the prometheus instance. *Default:* `http://localhost:9090` | -| --path | RESULT_PATH | A directory path for the results. Relative to the Execution folder. *Default:* `results` | -| --configurations | CONFIGURATIONS | Defines environment variables for the use cases and, thus, enables further configuration options. | -| --threshold | THRESHOLD | The threshold for the trend slop that the search strategies use to determine that a load could be handled. *Default:* `2000` | +Be sure, that the names of the configmap corresponds correctly to the specifications of the mounted `configmaps`, `volumes`, `mountPath`. In particular: The name of the execution file and the benchmark file must match the value of the corresponding environment variable. ### Domain Restriction diff --git a/execution/helm/.gitignore b/execution/helm/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..80bf7fc709ac6d08e703fe9f24d7d5776e26830e --- /dev/null +++ b/execution/helm/.gitignore @@ -0,0 +1 @@ +charts \ No newline at end of file diff --git a/execution/helm/templates/grafana/dashboard-config-map.yaml b/execution/helm/templates/grafana/dashboard-config-map.yaml index 1125d7833cc62e78c049436f38b854d926e2a216..41365e5efefaddc92a9f2f25f867a9d895e4ca3d 100644 --- a/execution/helm/templates/grafana/dashboard-config-map.yaml +++ b/execution/helm/templates/grafana/dashboard-config-map.yaml @@ -253,7 +253,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by(group, topic) (kafka_consumergroup_group_lag > 0)", + "expr": "sum by(group, topic) (kafka_consumergroup_group_lag >= 0)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{topic}}", @@ -436,7 +436,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by(group,topic) (kafka_consumergroup_group_offset > 0)", + "expr": "sum by(group,topic) (kafka_consumergroup_group_offset >= 0)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{topic}}", @@ -527,7 +527,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "count by(group,topic) (kafka_consumergroup_group_offset > 0)", + "expr": "count by(group,topic) (kafka_consumergroup_group_offset >= 0)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{topic}}", @@ -892,7 +892,7 @@ data: "steppedLine": false, "targets": [ { - "expr": "sum by(group) (kafka_consumergroup_group_lag > 0)", + "expr": "sum by(group) (kafka_consumergroup_group_lag >= 0)", "format": "time_series", "intervalFactor": 1, "legendFormat": "total lag", diff --git a/execution/helm/templates/theodolite/crd-benchmark.yaml b/execution/helm/templates/theodolite/crd-benchmark.yaml index 3244b9d196717b9865dd9c4e247f136fcf46d74b..f894f427332c1d4d5769d524fa27c17dfcdc1594 100644 --- a/execution/helm/templates/theodolite/crd-benchmark.yaml +++ b/execution/helm/templates/theodolite/crd-benchmark.yaml @@ -1,15 +1,119 @@ {{- if .Values.operator.benchmarkCRD.create -}} -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: benchmarks.theodolite.com spec: group: theodolite.com - version: v1alpha1 names: kind: benchmark plural: benchmarks + shortNames: + - bench + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + required: ["spec"] + properties: + spec: + type: object + required: [] + properties: + name: + type: string + appResource: + type: array + minItems: 1 + items: + type: string + loadGenResource: + type: array + minItems: 1 + items: + type: string + resourceTypes: + type: array + minItems: 1 + items: + type: object + properties: + typeName: + type: string + patchers: + type: array + minItems: 1 + items: + type: object + properties: + type: + type: string + default: "" + resource: + type: string + default: "" + properties: + type: object + additionalProperties: true + x-kubernetes-map-type: "granular" + default: {} + loadTypes: + type: array + minItems: 1 + items: + type: object + properties: + typeName: + type: string + patchers: + type: array + minItems: 1 + items: + type: object + properties: + type: + type: string + default: "" + resource: + type: string + default: "" + properties: + type: object + additionalProperties: true + x-kubernetes-map-type: "granular" + default: {} + kafkaConfig: + type: object + properties: + bootstrapServer: + type: string + topics: + type: array + minItems: 1 + items: + type: object + required: [] + properties: + name: + type: string + default: "" + numPartitions: + type: integer + default: 0 + replicationFactor: + type: integer + default: 0 + removeOnly: + type: boolean + default: false + additionalPrinterColumns: + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + subresources: + status: {} scope: Namespaced - subresources: - status: {} {{- end }} \ No newline at end of file diff --git a/execution/helm/templates/theodolite/crd-execution.yaml b/execution/helm/templates/theodolite/crd-execution.yaml index 5979cd5b60195e313831891a2007011c48cc76dd..f10fe3abf24d2609b2c2694c126fffd46a557365 100644 --- a/execution/helm/templates/theodolite/crd-execution.yaml +++ b/execution/helm/templates/theodolite/crd-execution.yaml @@ -1,15 +1,129 @@ {{- if .Values.operator.executionCRD.create -}} -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: executions.theodolite.com spec: group: theodolite.com - version: v1alpha1 names: kind: execution plural: executions - scope: Namespaced - subresources: - status: {} -{{- end }} \ No newline at end of file + shortNames: + - exec + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + required: ["spec"] + properties: + spec: + type: object + required: ["benchmark", "load", "resources", "slos", "execution", "configOverrides"] + properties: + name: + type: string + default: "" + benchmark: + type: string + load: # definition of the load dimension + type: object + required: ["loadType", "loadValues"] + properties: + loadType: + type: string + loadValues: + type: array + items: + type: integer + resources: # definition of the resource dimension + type: object + required: ["resourceType", "resourceValues"] + properties: + resourceType: + type: string + resourceValues: + type: array + items: + type: integer + slos: # def of service level objectives + type: array + items: + type: object + required: ["sloType", "threshold", "prometheusUrl", "externalSloUrl", "offset", "warmup"] + properties: + sloType: + type: string + threshold: + type: integer + prometheusUrl: + type: string + externalSloUrl: + type: string + offset: + type: integer + warmup: + type: integer + execution: # def execution config + type: object + required: ["strategy", "duration", "repetitions", "restrictions"] + properties: + strategy: + type: string + duration: + type: integer + repetitions: + type: integer + loadGenerationDelay: + type: integer + restrictions: + type: array + items: + type: string + configOverrides: + type: array + items: + type: object + properties: + patcher: + type: object + properties: + type: + type: string + default: "" + resource: + type: string + default: "" + properties: + type: object + additionalProperties: true + x-kubernetes-map-type: "granular" + default: {} + value: + type: string + status: + type: object + properties: + executionState: + description: "" + type: string + executionDuration: + description: "Duration of the execution in seconds" + type: string + additionalPrinterColumns: + - name: STATUS + type: string + description: State of the execution + jsonPath: .status.executionState + - name: Duration + type: string + description: Duration of the execution + jsonPath: .status.executionDuration + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + scope: Namespaced \ No newline at end of file diff --git a/execution/helm/templates/theodolite/random-scheduler/cluster-role-binding.yaml b/execution/helm/templates/theodolite/random-scheduler/cluster-role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..658f75c8c5018fe5b9f47cf9619bb4ee5b26b8e5 --- /dev/null +++ b/execution/helm/templates/theodolite/random-scheduler/cluster-role-binding.yaml @@ -0,0 +1,14 @@ +{{- if .Values.randomScheduler.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "theodolite.fullname" . }}-random-scheduler +subjects: +- kind: ServiceAccount + name: {{ include "theodolite.fullname" . }}-random-scheduler + namespace: kube-system +roleRef: + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io + name: system:kube-scheduler +{{- end }} diff --git a/execution/helm/templates/theodolite/random-scheduler/deployment.yaml b/execution/helm/templates/theodolite/random-scheduler/deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35a6ad027b93446a2bb97e2ebd67f2e27652e99a --- /dev/null +++ b/execution/helm/templates/theodolite/random-scheduler/deployment.yaml @@ -0,0 +1,30 @@ +{{- if .Values.randomScheduler.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "theodolite.fullname" . }}-random-scheduler + labels: + app: {{ include "theodolite.fullname" . }} + component: random-scheduler + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "theodolite.fullname" . }} + component: random-scheduler + template: + metadata: + labels: + app: {{ include "theodolite.fullname" . }} + component: random-scheduler + spec: + serviceAccount: {{ include "theodolite.fullname" . }}-random-scheduler + containers: + - name: random-scheduler + image: ghcr.io/cau-se/theodolite-random-scheduler:theodolite-kotlin-latest + #imagePullPolicy: Always + env: + - name: TARGET_NAMESPACE + value: {{ .Release.Namespace }} +{{- end }} diff --git a/execution/helm/templates/theodolite/random-scheduler/service-account.yaml b/execution/helm/templates/theodolite/random-scheduler/service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..babfff17b46d62e7e820fcb9dc8a35d73b4e6538 --- /dev/null +++ b/execution/helm/templates/theodolite/random-scheduler/service-account.yaml @@ -0,0 +1,10 @@ +{{- if .Values.randomScheduler.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: kube-system + name: {{ include "theodolite.fullname" . }}-random-scheduler + labels: + app: {{ include "theodolite.fullname" . }} + component: random-scheduler +{{- end }} diff --git a/execution/helm/templates/theodolite/role.yaml b/execution/helm/templates/theodolite/role.yaml index ee699a5b861c64e355ca2ba44330a7d600756b77..630a84c0491328a9bb1900c1f99a8f4643020e17 100644 --- a/execution/helm/templates/theodolite/role.yaml +++ b/execution/helm/templates/theodolite/role.yaml @@ -56,5 +56,14 @@ rules: - watch - update - patch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - delete + - get + - create + - update {{- end }} {{- end }} \ No newline at end of file diff --git a/execution/helm/templates/theodolite/thedolite-operator.yaml b/execution/helm/templates/theodolite/theodolite-operator.yaml similarity index 92% rename from execution/helm/templates/theodolite/thedolite-operator.yaml rename to execution/helm/templates/theodolite/theodolite-operator.yaml index ba63ce3487e1115cc314b4a09153fc5297e3a9b2..96a1a5a0119a10e3da16d1abcb0ebf3d3d015cb8 100644 --- a/execution/helm/templates/theodolite/thedolite-operator.yaml +++ b/execution/helm/templates/theodolite/theodolite-operator.yaml @@ -31,9 +31,13 @@ spec: mountPath: /work/benchmark-resources - name: lag-analysis image: ghcr.io/cau-se/theodolite-slo-checker-lag-trend:theodolite-kotlin-latest + imagePullPolicy: Always ports: - containerPort: 80 name: analysis + env: + - name: LOG_LEVEL + value: INFO volumes: - name: benchmark-resources configMap: diff --git a/execution/helm/values.yaml b/execution/helm/values.yaml index f4b585fac6351df2ace82d239590ce75bf1c194f..352334d546a058cfbe553006f1e21bdfa1cd4f4d 100644 --- a/execution/helm/values.yaml +++ b/execution/helm/values.yaml @@ -25,8 +25,16 @@ grafana: adminUser: admin adminPassword: admin grafana.ini: + #org_name: Theodolite + auth.anonymous: + # enable anonymous access + enabled: true + org_role: Admin # Role for unauthenticated users, other valid values are `Viewer`, `Editor` and `Admin` users: default_theme: light + #dashboards: # the following doesn't work but is planed + # Path to the default home dashboard. If this value is empty, then Grafana uses StaticRootPath + "dashboards/home.json" + #default_home_dashboard_path: "/tmp/dashboards/k8s-dashboard.json" ## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders ## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards sidecar: @@ -92,8 +100,9 @@ cp-helm-charts: "replica.fetch.max.bytes": "134217728" # 128 MB #default.replication.factor: 1 # "min.insync.replicas": 2 - # "auto.create.topics.enable": false - "log.retention.ms": "10000" # 10s + "auto.create.topics.enable": false + #"log.retention.ms": "10000" # 10s + "log.retention.ms": "7200000" # 2h "metrics.sample.window.ms": "5000" #5s "advertised.listeners": |- EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID})) @@ -251,3 +260,10 @@ serviceAccount: rbac: create: true + +randomScheduler: + enabled: true + rbac: + create: true + serviceAccount: + create: true diff --git a/execution/infrastructure/kubernetes/rbac/role.yaml b/execution/infrastructure/kubernetes/rbac/role.yaml index a21fd554f0f56b3955e9a9b6cf4bf95442b5d7af..e45814eedacd30715075f66e520f9f9e6bfc42ad 100644 --- a/execution/infrastructure/kubernetes/rbac/role.yaml +++ b/execution/infrastructure/kubernetes/rbac/role.yaml @@ -52,4 +52,12 @@ rules: - watch - update - patch - + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - delete + - get + - create + - update \ No newline at end of file diff --git a/execution/theodolite.yaml b/execution/theodolite.yaml index 06d14a0f589b2ac7a16ebaaae4d1490b840ea57b..ff8eecb312d052eab6f2e66a0bd57d8a983d38e1 100644 --- a/execution/theodolite.yaml +++ b/execution/theodolite.yaml @@ -5,47 +5,60 @@ metadata: spec: template: spec: - volumes: - - name: theodolite-pv-storage - persistentVolumeClaim: - claimName: theodolite-pv-claim + securityContext: + runAsUser: 0 # Set the permissions for write access to the volumes. containers: + - name: lag-analysis + image: ghcr.io/cau-se/theodolite-slo-checker-lag-trend:theodolite-kotlin-latest + ports: + - containerPort: 80 + name: analysis - name: theodolite - image: ghcr.io/cau-se/theodolite:latest - # imagePullPolicy: Never # Used to pull "own" local image + image: ghcr.io/cau-se/theodolite:theodolite-kotlin-latest + imagePullPolicy: Always env: - - name: UC # mandatory - value: "1" - - name: LOADS # mandatory - value: "100000, 200000" - - name: INSTANCES # mandatory - value: "1, 2, 3" - # - name: DURATION - # value: "5" - # - name: PARTITIONS - # value: "40" - # - name: DOMAIN_RESTRICTION - # value: "True" - # - name: SEARCH_STRATEGY - # value: "linear-search" - # - name: CPU_LIMIT - # value: "1000m" - # - name: MEMORY_LIMIT - # value: "4Gi" - - name: PROMETHEUS_BASE_URL - value: "http://prometheus-operated:9090" - # - name: NAMESPACE - # value: "default" - # - name: CONFIGURATIONS - # value: "COMMIT_INTERVAL_MS=100, NUM_STREAM_THREADS=1" - - name: RESULT_PATH - value: "results" - - name: PYTHONUNBUFFERED # Enable logs in Kubernetes - value: "1" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + + # - name: MODE + # value: yaml-executor # Default is `yaml-executor` + - name: THEODOLITE_EXECUTION + value: /etc/execution/example-execution-yaml-resource.yaml # The name of this file must correspond to the filename of the execution, from which the config map is created. + - name: THEODOLITE_BENCHMARK + value: /etc/benchmark/example-benchmark-yaml-resource.yaml # The name of this file must correspond to the filename of the benchmark, from which the config map is created. + - name: THEODOLITE_APP_RESOURCES + value: /etc/app-resources + - name: RESULTS_FOLDER # Folder for saving results + value: results # Default is the pwd (/deployments) + # - name: CREATE_RESULTS_FOLDER # Specify whether the specified result folder should be created if it does not exist. + # value: "false" # Default is false. volumeMounts: - - mountPath: "/app/results" + - mountPath: "/deployments/results" # the mounted path must corresponds to the value of `RESULT_FOLDER`. name: theodolite-pv-storage + - mountPath: "/etc/app-resources" # must be corresponds to the value of `THEODOLITE_APP_RESOURCES`. + name: app-resources + - mountPath: "/etc/benchmark" # must be corresponds to the value of `THEODOLITE_BENCHMARK`. + name: benchmark + - mountPath: "/etc/execution" # must be corresponds to the value of `THEODOLITE_EXECUTION`. + name: execution restartPolicy: Never # Uncomment if RBAC is enabled and configured - # serviceAccountName: theodolite - backoffLimit: 4 + serviceAccountName: theodolite + # Multiple volumes are needed to provide the corresponding files. + # The names must correspond to the created configmaps and the volumeMounts. + volumes: + - name: theodolite-pv-storage + persistentVolumeClaim: + claimName: theodolite-pv-claim + - name: app-resources + configMap: + name: app-resources-configmap + - name: benchmark + configMap: + name: benchmark-configmap + - name: execution + configMap: + name: execution-configmap + backoffLimit: 4 \ No newline at end of file diff --git a/slope-evaluator/README.md b/slope-evaluator/README.md index 69831cd5f83665735c586ab25493ed257b93c2ad..cd9e6820ed46452ce44d57d0c7e5cd5ae05e5a3b 100644 --- a/slope-evaluator/README.md +++ b/slope-evaluator/README.md @@ -5,10 +5,10 @@ For development: ```sh -uvicorn main:app --reload +uvicorn main:app --reload # run this command inside the app/ folder ``` -Build the docker image: +## Build the docker image: ```sh docker build . -t theodolite-evaluator @@ -17,13 +17,13 @@ docker build . -t theodolite-evaluator Run the Docker image: ```sh - docker run -p 80:80 theodolite-evaluator +docker run -p 80:80 theodolite-evaluator ``` ## Configuration You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`. -For more information see [here](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage). +For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage). ## API Documentation @@ -32,7 +32,7 @@ The running webserver provides a REST API with the following route: * /evaluate-slope * Method: POST * Body: - * total_lag + * total_lags * threshold * warmup @@ -40,14 +40,16 @@ The body of the request must be a JSON string that satisfies the following condi * **total_lag**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON structure: ``` - { - "metric": { - "group": "<label_value>" - }, - "values": [ - [ - <unix_timestamp>, - "<sample_value>" + { + [ + "metric": { + "group": "<label_value>" + }, + "values": [ + [ + <unix_timestamp>, + "<sample_value>" + ] ] ] } diff --git a/slope-evaluator/app/main.py b/slope-evaluator/app/main.py index 83709c0f71563d9bd1c29c5f064645144163ea72..6f6788f0ca84b7710be5b509ca4f0641047e963d 100644 --- a/slope-evaluator/app/main.py +++ b/slope-evaluator/app/main.py @@ -5,6 +5,7 @@ import os import pandas as pd import json import sys +from statistics import median app = FastAPI() @@ -18,9 +19,9 @@ if os.getenv('LOG_LEVEL') == 'INFO': elif os.getenv('LOG_LEVEL') == 'WARNING': logger.setLevel(logging.WARNING) elif os.getenv('LOG_LEVEL') == 'DEBUG': - logger.setLevel((logging.DEBUG)) + logger.setLevel(logging.DEBUG) -def execute(results, threshold, warmup): +def calculate_slope_trend(results, warmup): d = [] for result in results: group = result['metric']['group'] @@ -30,22 +31,25 @@ def execute(results, threshold, warmup): df = pd.DataFrame(d) - logger.info(df) + logger.info("Calculating trend slope with warmup of %s seconds for data frame:\n %s", warmup, df) try: trend_slope = trend_slope_computer.compute(df, warmup) except Exception as e: - err_msg = 'Computing trend slope failed' + err_msg = 'Computing trend slope failed.' logger.exception(err_msg) - logger.error('Mark this subexperiment as not successful and continue benchmark') + logger.error('Mark this subexperiment as not successful and continue benchmark.') return False - logger.info("Trend Slope: %s", trend_slope) + logger.info("Computed lag trend slope is '%s'", trend_slope) + return trend_slope - return trend_slope < threshold +def check_service_level_objective(results, threshold): + return median(results) < threshold @app.post("/evaluate-slope",response_model=bool) async def evaluate_slope(request: Request): data = json.loads(await request.body()) - return execute(data['total_lag'], data['threshold'], data['warmup']) + results = [calculate_slope_trend(total_lag, data['warmup']) for total_lag in data['total_lags']] + return check_service_level_objective(results=results, threshold=data["threshold"]) -logger.info("Slope evaluator is online") \ No newline at end of file +logger.info("SLO evaluator is online") \ No newline at end of file diff --git a/slope-evaluator/app/test.py b/slope-evaluator/app/test.py new file mode 100644 index 0000000000000000000000000000000000000000..9b165ea479bb9a552edaba7692df4fd4ef3f4ab4 --- /dev/null +++ b/slope-evaluator/app/test.py @@ -0,0 +1,30 @@ +import unittest +from main import app, check_service_level_objective +import json +from fastapi.testclient import TestClient + +class TestSloEvaluation(unittest.TestCase): + client = TestClient(app) + + def test_1_rep(self): + with open('../resources/test-1-rep-success.json') as json_file: + data = json.load(json_file) + response = self.client.post("/evaluate-slope", json=data) + self.assertEquals(response.json(), True) + + def test_3_rep(self): + with open('../resources/test-3-rep-success.json') as json_file: + data = json.load(json_file) + response = self.client.post("/evaluate-slope", json=data) + self.assertEquals(response.json(), True) + + def test_check_service_level_objective(self): + list = [1,2,3,4] + self.assertEquals(check_service_level_objective(list, 2), False) + self.assertEquals(check_service_level_objective(list, 3), True) + list = [1,2,3,4,5] + self.assertEquals(check_service_level_objective(list, 2), False) + self.assertEquals(check_service_level_objective(list, 4), True) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/slope-evaluator/app/trend_slope_computer.py b/slope-evaluator/app/trend_slope_computer.py index c128d9f48c1e7ba20e43dfbfd6a0391eeec2b60b..51b28f2baa5110a6d64f3adc1ac9a94c6b6f3ce9 100644 --- a/slope-evaluator/app/trend_slope_computer.py +++ b/slope-evaluator/app/trend_slope_computer.py @@ -2,13 +2,12 @@ from sklearn.linear_model import LinearRegression import pandas as pd import os -def compute(x, warmup_sec): - input = x - input['sec_start'] = input.loc[0:, 'timestamp'] - input.iloc[0]['timestamp'] - regress = input.loc[input['sec_start'] >= warmup_sec] # Warm-Up +def compute(data, warmup_sec): + data['sec_start'] = data.loc[0:, 'timestamp'] - data.iloc[0]['timestamp'] + regress = data.loc[data['sec_start'] >= warmup_sec] # Warm-Up - X = regress.iloc[:, 2].values.reshape(-1, 1) # values converts it into a numpy array - Y = regress.iloc[:, 3].values.reshape(-1, 1) # -1 means that calculate the dimension of rows, but have 1 column + X = regress.iloc[:, 1].values.reshape(-1, 1) # values converts it into a numpy array + Y = regress.iloc[:, 2].values.reshape(-1, 1) # -1 means that calculate the dimension of rows, but have 1 column linear_regressor = LinearRegression() # create object for the class linear_regressor.fit(X, Y) # perform linear regression Y_pred = linear_regressor.predict(X) # make predictions diff --git a/slope-evaluator/requirements.txt b/slope-evaluator/requirements.txt index ca77b6c891136b1388aaf56c5ae269d6ee4b5729..670815f35b18361951a2fa7b2142eee6bc86b01d 100644 --- a/slope-evaluator/requirements.txt +++ b/slope-evaluator/requirements.txt @@ -1,3 +1,5 @@ fastapi==0.55.1 scikit-learn==0.20.3 pandas==1.0.3 +uvicorn +requests diff --git a/slope-evaluator/resources/test-1-rep-success.json b/slope-evaluator/resources/test-1-rep-success.json new file mode 100644 index 0000000000000000000000000000000000000000..9e315c707be7b2a874c58fcb1093aa86f7676560 --- /dev/null +++ b/slope-evaluator/resources/test-1-rep-success.json @@ -0,0 +1,139 @@ +{ + "total_lags": [ + [ + { + "metric": { + "group": "theodolite-uc1-application-0.0.1" + }, + "values": [ + [ + 1.621008960827E9, + "234" + ], + [ + 1.621008965827E9, + "234" + ], + [ + 1.621008970827E9, + "234" + ], + [ + 1.621008975827E9, + "719" + ], + [ + 1.621008980827E9, + "719" + ], + [ + 1.621008985827E9, + "719" + ], + [ + 1.621008990827E9, + "1026" + ], + [ + 1.621008995827E9, + "1026" + ], + [ + 1.621009000827E9, + "1026" + ], + [ + 1.621009005827E9, + "534" + ], + [ + 1.621009010827E9, + "534" + ], + [ + 1.621009015827E9, + "534" + ], + [ + 1.621009020827E9, + "943" + ], + [ + 1.621009025827E9, + "943" + ], + [ + 1.621009030827E9, + "943" + ], + [ + 1.621009035827E9, + "66" + ], + [ + 1.621009040827E9, + "66" + ], + [ + 1.621009045827E9, + "66" + ], + [ + 1.621009050827E9, + "841" + ], + [ + 1.621009055827E9, + "841" + ], + [ + 1.621009060827E9, + "841" + ], + [ + 1.621009065827E9, + "405" + ], + [ + 1.621009070827E9, + "405" + ], + [ + 1.621009075827E9, + "405" + ], + [ + 1.621009080827E9, + "201" + ], + [ + 1.621009085827E9, + "201" + ], + [ + 1.621009090827E9, + "201" + ], + [ + 1.621009095827E9, + "227" + ], + [ + 1.621009100827E9, + "227" + ], + [ + 1.621009105827E9, + "227" + ], + [ + 1.621009110827E9, + "943" + ] + ] + } + ] + ], + "threshold": 2000, + "warmup": 0 +} \ No newline at end of file diff --git a/slope-evaluator/resources/test-3-rep-success.json b/slope-evaluator/resources/test-3-rep-success.json new file mode 100644 index 0000000000000000000000000000000000000000..485966cba40f01e4a646e626914510ba49b707bc --- /dev/null +++ b/slope-evaluator/resources/test-3-rep-success.json @@ -0,0 +1,289 @@ +{ + "total_lags": [ + [ + { + "metric": { + "group": "theodolite-uc1-application-0.0.1" + }, + "values": [ + [ + 1.621012384232E9, + "6073" + ], + [ + 1.621012389232E9, + "6073" + ], + [ + 1.621012394232E9, + "6073" + ], + [ + 1.621012399232E9, + "227" + ], + [ + 1.621012404232E9, + "227" + ], + [ + 1.621012409232E9, + "227" + ], + [ + 1.621012414232E9, + "987" + ], + [ + 1.621012419232E9, + "987" + ], + [ + 1.621012424232E9, + "987" + ], + [ + 1.621012429232E9, + "100" + ], + [ + 1.621012434232E9, + "100" + ], + [ + 1.621012439232E9, + "100" + ], + [ + 1.621012444232E9, + "959" + ], + [ + 1.621012449232E9, + "959" + ], + [ + 1.621012454232E9, + "959" + ], + [ + 1.621012459232E9, + "625" + ], + [ + 1.621012464232E9, + "625" + ], + [ + 1.621012469232E9, + "625" + ], + [ + 1.621012474232E9, + "683" + ], + [ + 1.621012479232E9, + "683" + ], + [ + 1.621012484232E9, + "683" + ], + [ + 1.621012489232E9, + "156" + ] + ] + } + ], + [ + { + "metric": { + "group": "theodolite-uc1-application-0.0.1" + }, + "values": [ + [ + 1.621012545211E9, + "446" + ], + [ + 1.621012550211E9, + "446" + ], + [ + 1.621012555211E9, + "446" + ], + [ + 1.621012560211E9, + "801" + ], + [ + 1.621012565211E9, + "801" + ], + [ + 1.621012570211E9, + "801" + ], + [ + 1.621012575211E9, + "773" + ], + [ + 1.621012580211E9, + "773" + ], + [ + 1.621012585211E9, + "773" + ], + [ + 1.621012590211E9, + "509" + ], + [ + 1.621012595211E9, + "509" + ], + [ + 1.621012600211E9, + "509" + ], + [ + 1.621012605211E9, + "736" + ], + [ + 1.621012610211E9, + "736" + ], + [ + 1.621012615211E9, + "736" + ], + [ + 1.621012620211E9, + "903" + ], + [ + 1.621012625211E9, + "903" + ], + [ + 1.621012630211E9, + "903" + ], + [ + 1.621012635211E9, + "512" + ], + [ + 1.621012640211E9, + "512" + ], + [ + 1.621012645211E9, + "512" + ] + ] + } + ], + [ + { + "metric": { + "group": "theodolite-uc1-application-0.0.1" + }, + "values": [ + [ + 1.621012700748E9, + "6484" + ], + [ + 1.621012705748E9, + "6484" + ], + [ + 1.621012710748E9, + "6484" + ], + [ + 1.621012715748E9, + "505" + ], + [ + 1.621012720748E9, + "505" + ], + [ + 1.621012725748E9, + "505" + ], + [ + 1.621012730748E9, + "103" + ], + [ + 1.621012735748E9, + "103" + ], + [ + 1.621012740748E9, + "103" + ], + [ + 1.621012745748E9, + "201" + ], + [ + 1.621012750748E9, + "201" + ], + [ + 1.621012755748E9, + "201" + ], + [ + 1.621012760748E9, + "965" + ], + [ + 1.621012765748E9, + "965" + ], + [ + 1.621012770748E9, + "965" + ], + [ + 1.621012775748E9, + "876" + ], + [ + 1.621012780748E9, + "876" + ], + [ + 1.621012785748E9, + "876" + ], + [ + 1.621012790748E9, + "380" + ], + [ + 1.621012795748E9, + "380" + ], + [ + 1.621012800748E9, + "380" + ] + ] + } + ] + ], + "threshold": 2000, + "warmup": 0 +} \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36178e2bebdac96b8648bd6c299009aa49d3fff6 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-flink/flink-configuration-configmap.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flink-config + labels: + app: flink +data: + flink-conf.yaml: |+ + jobmanager.rpc.address: flink-jobmanager + taskmanager.numberOfTaskSlots: 1 #TODO + #blob.server.port: 6124 + #jobmanager.rpc.port: 6123 + #taskmanager.rpc.port: 6122 + #queryable-state.proxy.ports: 6125 + #jobmanager.memory.process.size: 4Gb + #taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter + metrics.reporter.prom.interval: 10 SECONDS + taskmanager.network.detailed-metrics: true + # -> gives metrics about inbound/outbound network queue lengths + log4j-console.properties: |+ + # This affects logging for both user code and Flink + rootLogger.level = INFO + rootLogger.appenderRef.console.ref = ConsoleAppender + rootLogger.appenderRef.rolling.ref = RollingFileAppender + + # Uncomment this if you want to _only_ change Flink's logging + #logger.flink.name = org.apache.flink + #logger.flink.level = INFO + + # The following lines keep the log level of common libraries/connectors on + # log level INFO. The root logger does not override this. You have to manually + # change the log levels here. + logger.akka.name = akka + logger.akka.level = INFO + logger.kafka.name= org.apache.kafka + logger.kafka.level = INFO + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = INFO + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = INFO + + # Log all infos to the console + appender.console.name = ConsoleAppender + appender.console.type = CONSOLE + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + + # Log all infos in the given rolling file + appender.rolling.name = RollingFileAppender + appender.rolling.type = RollingFile + appender.rolling.append = false + appender.rolling.fileName = ${sys:log.file} + appender.rolling.filePattern = ${sys:log.file}.%i + appender.rolling.layout.type = PatternLayout + appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + appender.rolling.policies.type = Policies + appender.rolling.policies.size.type = SizeBasedTriggeringPolicy + appender.rolling.policies.size.size=100MB + appender.rolling.strategy.type = DefaultRolloverStrategy + appender.rolling.strategy.max = 10 + + # Suppress the irrelevant (wrong) warnings from the Netty channel handler + logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline + logger.netty.level = OFF \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..512d4fe3c786e1b2c44e6ec57fccadf41a2e2eeb --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-deployment.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-jobmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink + component: jobmanager + template: + metadata: + labels: + app: flink + component: jobmanager + spec: + containers: + - name: jobmanager + image: ghcr.io/cau-se/theodolite-uc1-flink:latest + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: COMMIT_INTERVAL_MS + value: "100" + - name: CHECKPOINTING + value: "false" + - name: PARALLELISM + value: "1" + - name: "FLINK_STATE_BACKEND" + value: "rocksdb" + - name: JOB_MANAGER_RPC_ADDRESS + value: "flink-jobmanager" + - name: FLINK_PROPERTIES + value: |+ + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + resources: + limits: + memory: 4Gi + cpu: 1000m + args: ["standalone-job", "--job-classname", "theodolite.uc1.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"] + #command: ['sleep', '60m'] + ports: + - containerPort: 6123 + name: rpc + - containerPort: 6124 + name: blob-server + - containerPort: 8081 + name: webui + - containerPort: 9249 + name: metrics + livenessProbe: + tcpSocket: + port: 6123 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume-rw + mountPath: /opt/flink/conf +# - name: job-artifacts-volume +# mountPath: /opt/flink/usrlib + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + initContainers: + - name: init-jobmanager + image: busybox:1.28 + command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] + volumeMounts: + - name: flink-config-volume + mountPath: /flink-config/ + - name: flink-config-volume-rw + mountPath: /flink-config-rw/ + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties + - name: flink-config-volume-rw + emptyDir: {} +# - name: job-artifacts-volume +# hostPath: +# path: /host/path/to/job/artifacts \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-rest-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager-rest +spec: + type: NodePort + ports: + - name: rest + port: 8081 + targetPort: 8081 + nodePort: 30081 + selector: + app: flink + component: jobmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-flink/jobmanager-service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager + labels: + app: flink +spec: + type: ClusterIP + ports: + - name: rpc + port: 6123 + - name: blob-server + port: 6124 + - name: webui + port: 8081 + - name: metrics + port: 9249 + selector: + app: flink + component: jobmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-flink/service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: flink + appScope: titan-ccp + name: flink +spec: + selector: + matchLabels: + app: flink + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d46554692696b194736df6023eed5686040497d --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-deployment.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-taskmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink + component: taskmanager + template: + metadata: + labels: + app: flink + component: taskmanager + spec: + containers: + - name: taskmanager + image: ghcr.io/cau-se/theodolite-uc1-flink:latest + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: COMMIT_INTERVAL_MS + value: "100" + - name: CHECKPOINTING + value: "false" + - name: PARALLELISM + value: "1" + - name: "FLINK_STATE_BACKEND" + value: "rocksdb" + - name: JOB_MANAGER_RPC_ADDRESS + value: "flink-jobmanager" + - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS + value: "1" #TODO + - name: FLINK_PROPERTIES + value: |+ + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + resources: + limits: + memory: 4Gi + cpu: 1000m + args: ["taskmanager"] + ports: + - containerPort: 6122 + name: rpc + - containerPort: 6125 + name: query-state + - containerPort: 9249 + name: metrics + livenessProbe: + tcpSocket: + port: 6122 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume-rw + mountPath: /opt/flink/conf/ + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + initContainers: + - name: init-taskmanager + image: busybox:1.28 + command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] + volumeMounts: + - name: flink-config-volume + mountPath: /flink-config/ + - name: flink-config-volume-rw + mountPath: /flink-config-rw/ + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties + - name: flink-config-volume-rw + emptyDir: {} diff --git a/theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-flink/taskmanager-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-taskmanager + labels: + app: flink +spec: + type: ClusterIP + ports: + - name: metrics + port: 9249 + selector: + app: flink + component: taskmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b80b572dfd30e9c056d3c01ba17cc662d70fc749 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-operator.yaml @@ -0,0 +1,38 @@ +apiVersion: theodolite.com/v1 +kind: benchmark +metadata: + name: uc1-kstreams +spec: + appResource: + - "uc1-kstreams-deployment.yaml" + - "uc1-kstreams-service.yaml" + - "uc1-jmx-configmap.yaml" + - "uc1-service-monitor.yaml" + loadGenResource: + - "uc1-load-generator-deployment.yaml" + - "uc1-load-generator-service.yaml" + resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc1-kstreams-deployment.yaml" + loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc1-load-generator-deployment.yaml" + properties: + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: "NumSensorsLoadGeneratorReplicaPatcher" + resource: "uc1-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" + kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml similarity index 75% rename from theodolite-quarkus/config/example-benchmark-yaml-resource.yaml rename to theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml index 60eb3bb9c31e3eab3e70f916b450372d56db4968..12cbd8ea310423d28e35de8185288b27257c15ec 100644 --- a/theodolite-quarkus/config/example-benchmark-yaml-resource.yaml +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-benchmark-standalone.yaml @@ -1,8 +1,8 @@ name: "uc1-kstreams" appResource: - "uc1-kstreams-deployment.yaml" - - "aggregation-service.yaml" - - "jmx-configmap.yaml" + - "uc1-kstreams-service.yaml" + - "uc1-jmx-configmap.yaml" - "uc1-service-monitor.yaml" loadGenResource: - "uc1-load-generator-deployment.yaml" @@ -19,6 +19,10 @@ loadTypes: resource: "uc1-load-generator-deployment.yaml" container: "workload-generator" variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc1-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" kafkaConfig: bootstrapServer: "theodolite-cp-kafka:9092" topics: diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-jmx-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78496a86b1242a89b9e844ead3e700fd0b9a9667 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-jmx-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aggregation-jmx-configmap +data: + jmx-kafka-prometheus.yml: |+ + jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..171c3446db2719ee91bd8954233015316851fcf9 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest + ports: + - containerPort: 5555 + name: jmx + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m + - name: prometheus-jmx-exporter + image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-aggregation/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-aggregation + volumes: + - name: jmx-config + configMap: + name: aggregation-jmx-configmap \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85432d04f225c30469f3232153ef6bd72bd02bdf --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-kstreams-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-aggregation + labels: + app: titan-ccp-aggregation +spec: + #type: NodePort + selector: + app: titan-ccp-aggregation + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: metrics + port: 5556 diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f9ccc6ae39407bb1f027e1e23cb152944b869e0 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-load-generator +spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-load-generator-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator +spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc1-kstreams/uc1-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: titan-ccp-aggregation + appScope: titan-ccp + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..321541f6ac8715b8546b964d8ad2b7c28552fbcd --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-flink/flink-configuration-configmap.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flink-config + labels: + app: flink +data: + flink-conf.yaml: |+ + #jobmanager.rpc.address: flink-jobmanager + #taskmanager.numberOfTaskSlots: 1 #TODO + #blob.server.port: 6124 + #jobmanager.rpc.port: 6123 + #taskmanager.rpc.port: 6122 + #queryable-state.proxy.ports: 6125 + #jobmanager.memory.process.size: 4Gb + #taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter + metrics.reporter.prom.interval: 10 SECONDS + taskmanager.network.detailed-metrics: true + # -> gives metrics about inbound/outbound network queue lengths + log4j-console.properties: |+ + # This affects logging for both user code and Flink + rootLogger.level = INFO + rootLogger.appenderRef.console.ref = ConsoleAppender + rootLogger.appenderRef.rolling.ref = RollingFileAppender + + # Uncomment this if you want to _only_ change Flink's logging + #logger.flink.name = org.apache.flink + #logger.flink.level = INFO + + # The following lines keep the log level of common libraries/connectors on + # log level INFO. The root logger does not override this. You have to manually + # change the log levels here. + logger.akka.name = akka + logger.akka.level = INFO + logger.kafka.name= org.apache.kafka + logger.kafka.level = INFO + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = INFO + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = INFO + + # Log all infos to the console + appender.console.name = ConsoleAppender + appender.console.type = CONSOLE + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + + # Log all infos in the given rolling file + appender.rolling.name = RollingFileAppender + appender.rolling.type = RollingFile + appender.rolling.append = false + appender.rolling.fileName = ${sys:log.file} + appender.rolling.filePattern = ${sys:log.file}.%i + appender.rolling.layout.type = PatternLayout + appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + appender.rolling.policies.type = Policies + appender.rolling.policies.size.type = SizeBasedTriggeringPolicy + appender.rolling.policies.size.size=100MB + appender.rolling.strategy.type = DefaultRolloverStrategy + appender.rolling.strategy.max = 10 + + # Suppress the irrelevant (wrong) warnings from the Netty channel handler + logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline + logger.netty.level = OFF \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cece4286d49a3f6ff139ca7f1e01c647acd5d9f3 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-deployment.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-jobmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink + component: jobmanager + template: + metadata: + labels: + app: flink + component: jobmanager + spec: + containers: + - name: jobmanager + image: ghcr.io/cau-se/theodolite-uc2-flink:latest + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: COMMIT_INTERVAL_MS + value: "100" + - name: CHECKPOINTING + value: "false" + - name: PARALLELISM + value: "1" + - name: "FLINK_STATE_BACKEND" + value: "rocksdb" + - name: JOB_MANAGER_RPC_ADDRESS + value: "flink-jobmanager" + - name: FLINK_PROPERTIES + value: |+ + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + resources: + limits: + memory: 4Gi + cpu: 1000m + args: ["standalone-job", "--job-classname", "theodolite.uc2.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"] + #command: ['sleep', '60m'] + ports: + - containerPort: 6123 + name: rpc + - containerPort: 6124 + name: blob-server + - containerPort: 8081 + name: webui + - containerPort: 9249 + name: metrics + livenessProbe: + tcpSocket: + port: 6123 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume-rw + mountPath: /opt/flink/conf +# - name: job-artifacts-volume +# mountPath: /opt/flink/usrlib + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + initContainers: + - name: init-jobmanager + image: busybox:1.28 + command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] + volumeMounts: + - name: flink-config-volume + mountPath: /flink-config/ + - name: flink-config-volume-rw + mountPath: /flink-config-rw/ + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties + - name: flink-config-volume-rw + emptyDir: {} +# - name: job-artifacts-volume +# hostPath: +# path: /host/path/to/job/artifacts \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-rest-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager-rest +spec: + type: NodePort + ports: + - name: rest + port: 8081 + targetPort: 8081 + nodePort: 30081 + selector: + app: flink + component: jobmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-flink/jobmanager-service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager + labels: + app: flink +spec: + type: ClusterIP + ports: + - name: rpc + port: 6123 + - name: blob-server + port: 6124 + - name: webui + port: 8081 + - name: metrics + port: 9249 + selector: + app: flink + component: jobmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-flink/service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: flink + appScope: titan-ccp + name: flink +spec: + selector: + matchLabels: + app: flink + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0a10f65aae92e4ac1fd8fb92bae97794c142232 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-deployment.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-taskmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink + component: taskmanager + template: + metadata: + labels: + app: flink + component: taskmanager + spec: + containers: + - name: taskmanager + image: ghcr.io/cau-se/theodolite-uc2-flink:latest + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: COMMIT_INTERVAL_MS + value: "100" + - name: CHECKPOINTING + value: "false" + - name: PARALLELISM + value: "1" + - name: "FLINK_STATE_BACKEND" + value: "rocksdb" + - name: JOB_MANAGER_RPC_ADDRESS + value: "flink-jobmanager" + - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS + value: "1" #TODO + - name: FLINK_PROPERTIES + value: |+ + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + resources: + limits: + memory: 4Gi + cpu: 1000m + args: ["taskmanager"] + ports: + - containerPort: 6122 + name: rpc + - containerPort: 6125 + name: query-state + - containerPort: 9249 + name: metrics + livenessProbe: + tcpSocket: + port: 6122 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume-rw + mountPath: /opt/flink/conf/ + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + initContainers: + - name: init-taskmanager + image: busybox:1.28 + command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] + volumeMounts: + - name: flink-config-volume + mountPath: /flink-config/ + - name: flink-config-volume-rw + mountPath: /flink-config-rw/ + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties + - name: flink-config-volume-rw + emptyDir: {} diff --git a/theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-flink/taskmanager-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-taskmanager + labels: + app: flink +spec: + type: ClusterIP + ports: + - name: metrics + port: 9249 + selector: + app: flink + component: taskmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b65bbdedb055c206c1ebcd7ab6a450318ee8c00f --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-operator.yaml @@ -0,0 +1,41 @@ +apiVersion: theodolite.com/v1 +kind: benchmark +spec: + metadata: + name: uc2-kstreams + appResource: + - "uc2-kstreams-deployment.yaml" + - "uc2-kstreams-service.yaml" + - "uc2-jmx-configmap.yaml" + - "uc2-service-monitor.yaml" + loadGenResource: + - "uc2-load-generator-deployment.yaml" + - "uc2-load-generator-service.yaml" + resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc2-kstreams-deployment.yaml" + loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc2-load-generator-deployment.yaml" + properties: + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc2-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" + kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 + - name: "output" + numPartitions: 40 + replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e38f83f5b05d05febb59c2f775a29b2d545acf0e --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-benchmark-standalone.yaml @@ -0,0 +1,37 @@ +name: "uc2-kstreams" +appResource: + - "uc2-kstreams-deployment.yaml" + - "uc2-kstreams-service.yaml" + - "uc2-jmx-configmap.yaml" + - "uc2-service-monitor.yaml" +loadGenResource: + - "uc2-load-generator-deployment.yaml" + - "uc2-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc2-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc2-load-generator-deployment.yaml" + properties: + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc2-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 + - name: "output" + numPartitions: 40 + replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78496a86b1242a89b9e844ead3e700fd0b9a9667 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-jmx-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aggregation-jmx-configmap +data: + jmx-kafka-prometheus.yml: |+ + jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e07bb3f9e536655712c06a004c5d1fb60ffa67e0 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc2-kstreams-app:latest + ports: + - containerPort: 5555 + name: jmx + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m + - name: prometheus-jmx-exporter + image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-aggregation/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-aggregation + volumes: + - name: jmx-config + configMap: + name: aggregation-jmx-configmap \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85432d04f225c30469f3232153ef6bd72bd02bdf --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-aggregation + labels: + app: titan-ccp-aggregation +spec: + #type: NodePort + selector: + app: titan-ccp-aggregation + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: metrics + port: 5556 diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfc0af71543c15b12b5c850919feb0e0a4f52f28 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-load-generator +spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc2-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-load-generator-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator +spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: titan-ccp-aggregation + appScope: titan-ccp + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..321541f6ac8715b8546b964d8ad2b7c28552fbcd --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-flink/flink-configuration-configmap.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flink-config + labels: + app: flink +data: + flink-conf.yaml: |+ + #jobmanager.rpc.address: flink-jobmanager + #taskmanager.numberOfTaskSlots: 1 #TODO + #blob.server.port: 6124 + #jobmanager.rpc.port: 6123 + #taskmanager.rpc.port: 6122 + #queryable-state.proxy.ports: 6125 + #jobmanager.memory.process.size: 4Gb + #taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter + metrics.reporter.prom.interval: 10 SECONDS + taskmanager.network.detailed-metrics: true + # -> gives metrics about inbound/outbound network queue lengths + log4j-console.properties: |+ + # This affects logging for both user code and Flink + rootLogger.level = INFO + rootLogger.appenderRef.console.ref = ConsoleAppender + rootLogger.appenderRef.rolling.ref = RollingFileAppender + + # Uncomment this if you want to _only_ change Flink's logging + #logger.flink.name = org.apache.flink + #logger.flink.level = INFO + + # The following lines keep the log level of common libraries/connectors on + # log level INFO. The root logger does not override this. You have to manually + # change the log levels here. + logger.akka.name = akka + logger.akka.level = INFO + logger.kafka.name= org.apache.kafka + logger.kafka.level = INFO + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = INFO + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = INFO + + # Log all infos to the console + appender.console.name = ConsoleAppender + appender.console.type = CONSOLE + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + + # Log all infos in the given rolling file + appender.rolling.name = RollingFileAppender + appender.rolling.type = RollingFile + appender.rolling.append = false + appender.rolling.fileName = ${sys:log.file} + appender.rolling.filePattern = ${sys:log.file}.%i + appender.rolling.layout.type = PatternLayout + appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + appender.rolling.policies.type = Policies + appender.rolling.policies.size.type = SizeBasedTriggeringPolicy + appender.rolling.policies.size.size=100MB + appender.rolling.strategy.type = DefaultRolloverStrategy + appender.rolling.strategy.max = 10 + + # Suppress the irrelevant (wrong) warnings from the Netty channel handler + logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline + logger.netty.level = OFF \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33bf1f1121a9764785db7a504799314a7ed40cf3 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-deployment.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-jobmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink + component: jobmanager + template: + metadata: + labels: + app: flink + component: jobmanager + spec: + containers: + - name: jobmanager + image: ghcr.io/cau-se/theodolite-uc3-flink:latest + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: COMMIT_INTERVAL_MS + value: "100" + - name: CHECKPOINTING + value: "false" + - name: PARALLELISM + value: "1" + - name: "FLINK_STATE_BACKEND" + value: "rocksdb" + - name: JOB_MANAGER_RPC_ADDRESS + value: "flink-jobmanager" + - name: FLINK_PROPERTIES + value: |+ + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + resources: + limits: + memory: 4Gi + cpu: 1000m + args: ["standalone-job", "--job-classname", "theodolite.uc3.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"] + #command: ['sleep', '60m'] + ports: + - containerPort: 6123 + name: rpc + - containerPort: 6124 + name: blob-server + - containerPort: 8081 + name: webui + - containerPort: 9249 + name: metrics + livenessProbe: + tcpSocket: + port: 6123 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume-rw + mountPath: /opt/flink/conf +# - name: job-artifacts-volume +# mountPath: /opt/flink/usrlib + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + initContainers: + - name: init-jobmanager + image: busybox:1.28 + command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] + volumeMounts: + - name: flink-config-volume + mountPath: /flink-config/ + - name: flink-config-volume-rw + mountPath: /flink-config-rw/ + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties + - name: flink-config-volume-rw + emptyDir: {} +# - name: job-artifacts-volume +# hostPath: +# path: /host/path/to/job/artifacts \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-rest-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager-rest +spec: + type: NodePort + ports: + - name: rest + port: 8081 + targetPort: 8081 + nodePort: 30081 + selector: + app: flink + component: jobmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-flink/jobmanager-service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager + labels: + app: flink +spec: + type: ClusterIP + ports: + - name: rpc + port: 6123 + - name: blob-server + port: 6124 + - name: webui + port: 8081 + - name: metrics + port: 9249 + selector: + app: flink + component: jobmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-flink/service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: flink + appScope: titan-ccp + name: flink +spec: + selector: + matchLabels: + app: flink + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f70b7308429f79cfd8f8bda7a7a96e2bc8d8689 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-deployment.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-taskmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink + component: taskmanager + template: + metadata: + labels: + app: flink + component: taskmanager + spec: + containers: + - name: taskmanager + image: ghcr.io/cau-se/theodolite-uc3-flink:latest + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: COMMIT_INTERVAL_MS + value: "100" + - name: CHECKPOINTING + value: "false" + - name: PARALLELISM + value: "1" + - name: "FLINK_STATE_BACKEND" + value: "rocksdb" + - name: JOB_MANAGER_RPC_ADDRESS + value: "flink-jobmanager" + - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS + value: "1" #TODO + - name: FLINK_PROPERTIES + value: |+ + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + resources: + limits: + memory: 4Gi + cpu: 1000m + args: ["taskmanager"] + ports: + - containerPort: 6122 + name: rpc + - containerPort: 6125 + name: query-state + - containerPort: 9249 + name: metrics + livenessProbe: + tcpSocket: + port: 6122 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume-rw + mountPath: /opt/flink/conf/ + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + initContainers: + - name: init-taskmanager + image: busybox:1.28 + command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] + volumeMounts: + - name: flink-config-volume + mountPath: /flink-config/ + - name: flink-config-volume-rw + mountPath: /flink-config-rw/ + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties + - name: flink-config-volume-rw + emptyDir: {} diff --git a/theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-flink/taskmanager-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-taskmanager + labels: + app: flink +spec: + type: ClusterIP + ports: + - name: metrics + port: 9249 + selector: + app: flink + component: taskmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfbd7191c5f4a315db29100bcc05341f88cffec2 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-operator.yaml @@ -0,0 +1,41 @@ +apiVersion: theodolite.com/v1 +kind: benchmark +spec: + metadata: + name: uc3-kstreams + appResource: + - "uc3-kstreams-deployment.yaml" + - "uc3-kstreams-service.yaml" + - "uc3-jmx-configmap.yaml" + - "uc3-service-monitor.yaml" + loadGenResource: + - "uc3-load-generator-deployment.yaml" + - "uc3-load-generator-service.yaml" + resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc3-kstreams-deployment.yaml" + loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc3-load-generator-deployment.yaml" + properties: + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc3-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" + kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 + - name: "output" + numPartitions: 40 + replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e00c1672c4a5a02128c2618b525573a4cddd6c72 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-benchmark-standalone.yaml @@ -0,0 +1,37 @@ +name: "uc3-kstreams" +appResource: + - "uc3-kstreams-deployment.yaml" + - "uc3-kstreams-service.yaml" + - "uc3-jmx-configmap.yaml" + - "uc3-service-monitor.yaml" +loadGenResource: + - "uc3-load-generator-deployment.yaml" + - "uc3-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc3-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc3-load-generator-deployment.yaml" + properties: + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumSensorsLoadGeneratorReplicaPatcher + resource: "uc3-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 + - name: "output" + numPartitions: 40 + replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78496a86b1242a89b9e844ead3e700fd0b9a9667 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-jmx-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aggregation-jmx-configmap +data: + jmx-kafka-prometheus.yml: |+ + jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3f63fae9e245e6116e0fe451480d9bc74b36433 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc3-kstreams-app:latest + ports: + - containerPort: 5555 + name: jmx + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m + - name: prometheus-jmx-exporter + image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-aggregation/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-aggregation + volumes: + - name: jmx-config + configMap: + name: aggregation-jmx-configmap \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85432d04f225c30469f3232153ef6bd72bd02bdf --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-kstreams-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-aggregation + labels: + app: titan-ccp-aggregation +spec: + #type: NodePort + selector: + app: titan-ccp-aggregation + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: metrics + port: 5556 diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1cad0b70fd82a5bbb43792ee79f9cf5cc71d95f --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-load-generator +spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc3-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-load-generator-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator +spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc3-kstreams/uc3-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: titan-ccp-aggregation + appScope: titan-ccp + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml b/theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..321541f6ac8715b8546b964d8ad2b7c28552fbcd --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-flink/flink-configuration-configmap.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flink-config + labels: + app: flink +data: + flink-conf.yaml: |+ + #jobmanager.rpc.address: flink-jobmanager + #taskmanager.numberOfTaskSlots: 1 #TODO + #blob.server.port: 6124 + #jobmanager.rpc.port: 6123 + #taskmanager.rpc.port: 6122 + #queryable-state.proxy.ports: 6125 + #jobmanager.memory.process.size: 4Gb + #taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter + metrics.reporter.prom.interval: 10 SECONDS + taskmanager.network.detailed-metrics: true + # -> gives metrics about inbound/outbound network queue lengths + log4j-console.properties: |+ + # This affects logging for both user code and Flink + rootLogger.level = INFO + rootLogger.appenderRef.console.ref = ConsoleAppender + rootLogger.appenderRef.rolling.ref = RollingFileAppender + + # Uncomment this if you want to _only_ change Flink's logging + #logger.flink.name = org.apache.flink + #logger.flink.level = INFO + + # The following lines keep the log level of common libraries/connectors on + # log level INFO. The root logger does not override this. You have to manually + # change the log levels here. + logger.akka.name = akka + logger.akka.level = INFO + logger.kafka.name= org.apache.kafka + logger.kafka.level = INFO + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = INFO + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = INFO + + # Log all infos to the console + appender.console.name = ConsoleAppender + appender.console.type = CONSOLE + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + + # Log all infos in the given rolling file + appender.rolling.name = RollingFileAppender + appender.rolling.type = RollingFile + appender.rolling.append = false + appender.rolling.fileName = ${sys:log.file} + appender.rolling.filePattern = ${sys:log.file}.%i + appender.rolling.layout.type = PatternLayout + appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + appender.rolling.policies.type = Policies + appender.rolling.policies.size.type = SizeBasedTriggeringPolicy + appender.rolling.policies.size.size=100MB + appender.rolling.strategy.type = DefaultRolloverStrategy + appender.rolling.strategy.max = 10 + + # Suppress the irrelevant (wrong) warnings from the Netty channel handler + logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline + logger.netty.level = OFF \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6533a2c4355e227a16aeface2080253bce19958 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-deployment.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-jobmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink + component: jobmanager + template: + metadata: + labels: + app: flink + component: jobmanager + spec: + containers: + - name: jobmanager + image: ghcr.io/cau-se/theodolite-uc4-flink:latest + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: COMMIT_INTERVAL_MS + value: "100" + - name: CHECKPOINTING + value: "false" + - name: PARALLELISM + value: "1" + - name: "FLINK_STATE_BACKEND" + value: "rocksdb" + - name: JOB_MANAGER_RPC_ADDRESS + value: "flink-jobmanager" + - name: FLINK_PROPERTIES + value: |+ + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + resources: + limits: + memory: 4Gi + cpu: 1000m + args: ["standalone-job", "--job-classname", "theodolite.uc4.application.AggregationServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"] + #command: ['sleep', '60m'] + ports: + - containerPort: 6123 + name: rpc + - containerPort: 6124 + name: blob-server + - containerPort: 8081 + name: webui + - containerPort: 9249 + name: metrics + livenessProbe: + tcpSocket: + port: 6123 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume-rw + mountPath: /opt/flink/conf +# - name: job-artifacts-volume +# mountPath: /opt/flink/usrlib + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + initContainers: + - name: init-jobmanager + image: busybox:1.28 + command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] + volumeMounts: + - name: flink-config-volume + mountPath: /flink-config/ + - name: flink-config-volume-rw + mountPath: /flink-config-rw/ + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties + - name: flink-config-volume-rw + emptyDir: {} +# - name: job-artifacts-volume +# hostPath: +# path: /host/path/to/job/artifacts \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d74aaf7f625c6922e2e1b4f20c19e50a39b68ac --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-rest-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager-rest +spec: + type: NodePort + ports: + - name: rest + port: 8081 + targetPort: 8081 + nodePort: 30081 + selector: + app: flink + component: jobmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2ff5d9898eb1ebf5db9a827472a47514ab1473c --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-flink/jobmanager-service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager + labels: + app: flink +spec: + type: ClusterIP + ports: + - name: rpc + port: 6123 + - name: blob-server + port: 6124 + - name: webui + port: 8081 + - name: metrics + port: 9249 + selector: + app: flink + component: jobmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml b/theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02f78823c627e27ddfe1db5eac3f6a7f7a7f1bf8 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-flink/service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: flink + appScope: titan-ccp + name: flink +spec: + selector: + matchLabels: + app: flink + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7363b013b21ad29b481e449113ccf31538505634 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-deployment.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-taskmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink + component: taskmanager + template: + metadata: + labels: + app: flink + component: taskmanager + spec: + containers: + - name: taskmanager + image: ghcr.io/cau-se/theodolite-uc4-flink:latest + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: COMMIT_INTERVAL_MS + value: "100" + - name: CHECKPOINTING + value: "false" + - name: PARALLELISM + value: "1" + - name: "FLINK_STATE_BACKEND" + value: "rocksdb" + - name: JOB_MANAGER_RPC_ADDRESS + value: "flink-jobmanager" + - name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS + value: "1" #TODO + - name: FLINK_PROPERTIES + value: |+ + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + queryable-state.proxy.ports: 6125 + jobmanager.memory.process.size: 4Gb + taskmanager.memory.process.size: 4Gb + #parallelism.default: 1 #TODO + resources: + limits: + memory: 4Gi + cpu: 1000m + args: ["taskmanager"] + ports: + - containerPort: 6122 + name: rpc + - containerPort: 6125 + name: query-state + - containerPort: 9249 + name: metrics + livenessProbe: + tcpSocket: + port: 6122 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume-rw + mountPath: /opt/flink/conf/ + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + initContainers: + - name: init-taskmanager + image: busybox:1.28 + command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/'] + volumeMounts: + - name: flink-config-volume + mountPath: /flink-config/ + - name: flink-config-volume-rw + mountPath: /flink-config-rw/ + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties + - name: flink-config-volume-rw + emptyDir: {} diff --git a/theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2e27f64af1cfd1a26da142b8a50bb41c8ba5fcb --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-flink/taskmanager-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-taskmanager + labels: + app: flink +spec: + type: ClusterIP + ports: + - name: metrics + port: 9249 + selector: + app: flink + component: taskmanager \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ed48c9afd0d8d02493f7afc2df3e440d0ffabdd --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-operator.yaml @@ -0,0 +1,48 @@ +apiVersion: theodolite.com/v1 +kind: benchmark +spec: + metadata: + name: uc4-kstreams + appResource: + - "uc4-kstreams-deployment.yaml" + - "uc4-kstreams-service.yaml" + - "uc4-jmx-configmap.yaml" + - "uc4-service-monitor.yaml" + loadGenResource: + - "uc4-load-generator-deployment.yaml" + - "uc4-load-generator-service.yaml" + resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc4-kstreams-deployment.yaml" + loadTypes: + - typeName: "NumNestedGroups" + patchers: + - type: "EnvVarPatcher" + resource: "uc4-load-generator-deployment.yaml" + properties: + container: "workload-generator" + variableName: "NUM_SENSORS" + - type: NumNestedGroupsLoadGeneratorReplicaPatcher + resource: "uc4-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" + numSensors: "4.0" + kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 + - name: "output" + numPartitions: 40 + replicationFactor: 1 + - name: "configuration" + numPartitions: 40 + replicationFactor: 1 + - name: "aggregation-feedback" + numPartitions: 40 + replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96e72c9b6d726267044464cce6deb32f60442e96 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-benchmark-standalone.yaml @@ -0,0 +1,44 @@ +name: "uc4-kstreams" +appResource: + - "uc4-kstreams-deployment.yaml" + - "uc4-kstreams-service.yaml" + - "uc4-jmx-configmap.yaml" + - "uc4-service-monitor.yaml" +loadGenResource: + - "uc4-load-generator-deployment.yaml" + - "uc4-load-generator-service.yaml" +resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc4-kstreams-deployment.yaml" +loadTypes: + - typeName: "NumNestedGroups" + patchers: + - type: "EnvVarPatcher" + resource: "uc4-load-generator-deployment.yaml" + properties: + container: "workload-generator" + variableName: "NUM_NESTED_GROUPS" + - type: "NumNestedGroupsLoadGeneratorReplicaPatcher" + resource: "uc4-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" + numSensors: "4.0" +kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 + - name: "output" + numPartitions: 40 + replicationFactor: 1 + - name: "configuration" + numPartitions: 40 + replicationFactor: 1 + - name: "aggregation-feedback" + numPartitions: 40 + replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78496a86b1242a89b9e844ead3e700fd0b9a9667 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-jmx-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: aggregation-jmx-configmap +data: + jmx-kafka-prometheus.yml: |+ + jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi + lowercaseOutputName: true + lowercaseOutputLabelNames: true + ssl: false diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20e0872d262df46b5c213d9d529983f5f4155735 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-aggregation + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: uc-application + image: ghcr.io/cau-se/theodolite-uc4-kstreams-app:latest + ports: + - containerPort: 5555 + name: jmx + env: + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: JAVA_OPTS + value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" + - name: COMMIT_INTERVAL_MS # Set as default for the applications + value: "100" + resources: + limits: + memory: 4Gi + cpu: 1000m + - name: prometheus-jmx-exporter + image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143" + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-aggregation/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-aggregation + volumes: + - name: jmx-config + configMap: + name: aggregation-jmx-configmap \ No newline at end of file diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85432d04f225c30469f3232153ef6bd72bd02bdf --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-aggregation + labels: + app: titan-ccp-aggregation +spec: + #type: NodePort + selector: + app: titan-ccp-aggregation + ports: + - name: http + port: 80 + targetPort: 80 + protocol: TCP + - name: metrics + port: 5556 diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a69d13daae57b06c77f316da9aa953b21ac096b --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-deployment.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: titan-ccp-load-generator +spec: + selector: + matchLabels: + app: titan-ccp-load-generator + replicas: 1 + template: + metadata: + labels: + app: titan-ccp-load-generator + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: workload-generator + image: ghcr.io/cau-se/theodolite-uc4-workload-generator:latest + ports: + - containerPort: 5701 + name: coordination + env: + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_DNS_NAME + value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local" + - name: KAFKA_BOOTSTRAP_SERVERS + value: "theodolite-cp-kafka:9092" + - name: SCHEMA_REGISTRY_URL + value: "http://theodolite-cp-schema-registry:8081" + - name: NUM_NESTED_GROUPS + value: "5" diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8b26b3f6dece427f9c1ad4db94e351b042749b3 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-load-generator-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: titan-ccp-load-generator + labels: + app: titan-ccp-load-generator +spec: + type: ClusterIP + clusterIP: None + selector: + app: titan-ccp-load-generator + ports: + - name: coordination + port: 5701 + targetPort: 5701 + protocol: TCP diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e7e758cacb5086305efa26292ddef2afc958096 --- /dev/null +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: titan-ccp-aggregation + appScope: titan-ccp + name: titan-ccp-aggregation +spec: + selector: + matchLabels: + app: titan-ccp-aggregation + endpoints: + - port: metrics + interval: 10s diff --git a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/ConfigurationKeys.java b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/ConfigurationKeys.java index ed961bab733a409dc07b1be7fa35562103c3e2f4..382525cfe75f82dbbe8fbcc85308b0e7788a43bc 100644 --- a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/ConfigurationKeys.java +++ b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/ConfigurationKeys.java @@ -19,6 +19,8 @@ public final class ConfigurationKeys { public static final String CHECKPOINTING = "checkpointing"; + public static final String PARALLELISM = "parallelism"; + private ConfigurationKeys() {} } diff --git a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java index 6655b52ec3020f46bb8a37c7124ee870fa663573..0cb132e526486e71409736b843dd25bdfa52da4a 100644 --- a/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java +++ b/theodolite-benchmarks/uc1-flink/src/main/java/theodolite/uc1/application/HistoryServiceFlinkJob.java @@ -1,6 +1,7 @@ package theodolite.uc1.application; import org.apache.commons.configuration2.Configuration; +import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; @@ -42,6 +43,14 @@ public final class HistoryServiceFlinkJob { if (checkpointing) { this.env.enableCheckpointing(commitIntervalMs); } + + // Parallelism + final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null); + if (parallelism != null) { + LOGGER.info("Set parallelism: {}.", parallelism); + this.env.setParallelism(parallelism); + } + } private void buildPipeline() { @@ -59,9 +68,10 @@ public final class HistoryServiceFlinkJob { final DataStream<ActivePowerRecord> stream = this.env.addSource(kafkaConsumer); stream - .rebalance() + // .rebalance() .map(new GsonMapper()) - .flatMap((record, c) -> LOGGER.info("Record: {}", record)); + .flatMap((record, c) -> LOGGER.info("Record: {}", record)) + .returns(Types.GENERIC(Object.class)); // Will never be used } /** diff --git a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/ConfigurationKeys.java b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/ConfigurationKeys.java index 9ba56c828a0ae5c6147aadd90d449c7cf2324992..e8261062689ce4c586a4e6fbde02878a28f48e97 100644 --- a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/ConfigurationKeys.java +++ b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/ConfigurationKeys.java @@ -30,6 +30,8 @@ public final class ConfigurationKeys { public static final String CHECKPOINTING = "checkpointing"; + public static final String PARALLELISM = "parallelism"; + private ConfigurationKeys() {} } diff --git a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java index b8452847df800226ad481f9309323a2a9a532939..d156d895d86bb01a31f96e08764df8b8df743c4d 100644 --- a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java +++ b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java @@ -56,6 +56,13 @@ public final class HistoryServiceFlinkJob { this.env.enableCheckpointing(commitIntervalMs); } + // Parallelism + final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null); + if (parallelism != null) { + LOGGER.info("Set parallelism: {}.", parallelism); + this.env.setParallelism(parallelism); + } + // State Backend final StateBackend stateBackend = StateBackends.fromConfiguration(this.config); this.env.setStateBackend(stateBackend); @@ -76,7 +83,9 @@ public final class HistoryServiceFlinkJob { final String schemaRegistryUrl = this.config.getString(ConfigurationKeys.SCHEMA_REGISTRY_URL); final String inputTopic = this.config.getString(ConfigurationKeys.KAFKA_INPUT_TOPIC); final String outputTopic = this.config.getString(ConfigurationKeys.KAFKA_OUTPUT_TOPIC); - final int windowDuration = this.config.getInt(ConfigurationKeys.KAFKA_WINDOW_DURATION_MINUTES); + final int windowDurationMinutes = + this.config.getInt(ConfigurationKeys.KAFKA_WINDOW_DURATION_MINUTES); + final Time windowDuration = Time.minutes(windowDurationMinutes); final boolean checkpointing = this.config.getBoolean(ConfigurationKeys.CHECKPOINTING, true); final KafkaConnectorFactory kafkaConnector = new KafkaConnectorFactory( @@ -93,9 +102,9 @@ public final class HistoryServiceFlinkJob { this.env .addSource(kafkaSource).name("[Kafka Consumer] Topic: " + inputTopic) - .rebalance() + // .rebalance() .keyBy(ActivePowerRecord::getIdentifier) - .window(TumblingEventTimeWindows.of(Time.minutes(windowDuration))) + .window(TumblingEventTimeWindows.of(windowDuration)) .aggregate(new StatsAggregateFunction(), new StatsProcessWindowFunction()) .map(t -> { final String key = t.f0; diff --git a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/ConfigurationKeys.java b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/ConfigurationKeys.java index a895c74d89c5d788c47b3b78dc70500b4b5a6f5b..bc4e0b9d2d230026e9d2b6df0a11e4fb68380aed 100644 --- a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/ConfigurationKeys.java +++ b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/ConfigurationKeys.java @@ -34,6 +34,8 @@ public final class ConfigurationKeys { public static final String CHECKPOINTING = "checkpointing"; + public static final String PARALLELISM = "parallelism"; + private ConfigurationKeys() {} } diff --git a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java index 0f26d37652924a16be1840fd759b3cd5b023f338..091b25674a2a31671ca68bd2076c694da9533d77 100644 --- a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java +++ b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java @@ -63,6 +63,13 @@ public final class HistoryServiceFlinkJob { this.env.enableCheckpointing(commitIntervalMs); } + // Parallelism + final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null); + if (parallelism != null) { + LOGGER.error("Set parallelism: {}.", parallelism); + this.env.setParallelism(parallelism); + } + // State Backend final StateBackend stateBackend = StateBackends.fromConfiguration(this.config); this.env.setStateBackend(stateBackend); @@ -110,9 +117,8 @@ public final class HistoryServiceFlinkJob { // Streaming topology final StatsKeyFactory<HourOfDayKey> keyFactory = new HourOfDayKeyFactory(); this.env - .addSource(kafkaSource) - .name("[Kafka Consumer] Topic: " + inputTopic) - .rebalance() + .addSource(kafkaSource).name("[Kafka Consumer] Topic: " + inputTopic) + // .rebalance() .keyBy((KeySelector<ActivePowerRecord, HourOfDayKey>) record -> { final Instant instant = Instant.ofEpochMilli(record.getTimestamp()); final LocalDateTime dateTime = LocalDateTime.ofInstant(instant, timeZone); diff --git a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java index 0db5a3d524f74fbf22304e8f9b44fa55eead321a..3e2878a893057024de00333492462f5029eb6d77 100644 --- a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java +++ b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java @@ -76,6 +76,13 @@ public final class AggregationServiceFlinkJob { this.env.enableCheckpointing(commitIntervalMs); } + // Parallelism + final Integer parallelism = this.config.getInteger(ConfigurationKeys.PARALLELISM, null); + if (parallelism != null) { + LOGGER.info("Set parallelism: {}.", parallelism); + this.env.setParallelism(parallelism); + } + // State Backend final StateBackend stateBackend = StateBackends.fromConfiguration(this.config); this.env.setStateBackend(stateBackend); @@ -145,7 +152,7 @@ public final class AggregationServiceFlinkJob { // Build input stream final DataStream<ActivePowerRecord> inputStream = this.env.addSource(kafkaInputSource) .name("[Kafka Consumer] Topic: " + inputTopic)// NOCS - .rebalance() + // .rebalance() .map(r -> r) .name("[Map] Rebalance Forward"); @@ -153,7 +160,7 @@ public final class AggregationServiceFlinkJob { final DataStream<ActivePowerRecord> aggregationsInputStream = this.env.addSource(kafkaOutputSource) .name("[Kafka Consumer] Topic: " + outputTopic) // NOCS - .rebalance() + // .rebalance() .map(r -> new ActivePowerRecord(r.getIdentifier(), r.getTimestamp(), r.getSumInW())) .name("[Map] AggregatedActivePowerRecord -> ActivePowerRecord"); diff --git a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/ConfigurationKeys.java b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/ConfigurationKeys.java index 6497f6b055ef115c4a681499c5fa38657bb5d29e..448e8b095ef15c434655ca3c76a9e2de21244054 100644 --- a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/ConfigurationKeys.java +++ b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/ConfigurationKeys.java @@ -15,7 +15,7 @@ public final class ConfigurationKeys { public static final String KAFKA_OUTPUT_TOPIC = "kafka.output.topic"; public static final String KAFKA_INPUT_TOPIC = "kafka.input.topic"; - + public static final String SCHEMA_REGISTRY_URL = "schema.registry.url"; public static final String WINDOW_SIZE_MS = "window.size.ms"; @@ -28,13 +28,15 @@ public final class ConfigurationKeys { public static final String FLINK_STATE_BACKEND_PATH = "flink.state.backend.path"; - public static final String FLINK_STATE_BACKEND_MEMORY_SIZE = //NOPMD + public static final String FLINK_STATE_BACKEND_MEMORY_SIZE = // NOPMD "flink.state.backend.memory.size"; public static final String DEBUG = "debug"; public static final String CHECKPOINTING = "checkpointing"; + public static final String PARALLELISM = "parallelism"; + private ConfigurationKeys() {} } diff --git a/theodolite-quarkus/README.md b/theodolite-quarkus/README.md index bb99f34e80f8b61b42f9b10b4c1d988871b74cb0..fc1d1bfe4a9c20a515cf6e69208657f74694d80e 100644 --- a/theodolite-quarkus/README.md +++ b/theodolite-quarkus/README.md @@ -1,4 +1,4 @@ -# theodolite-quarkus project +# Theodolite-quarkus project This project uses Quarkus, the Supersonic Subatomic Java Framework. @@ -6,7 +6,8 @@ If you want to learn more about Quarkus, please visit its website: https://quark ## Running the application in dev mode -You can run your application in dev mode that enables live coding using: +You can run your application in dev mode using: + ```shell script ./gradlew quarkusDev ``` @@ -14,13 +15,16 @@ You can run your application in dev mode that enables live coding using: ## Packaging and running the application The application can be packaged using: + ```shell script ./gradlew build ``` -It produces the `theodolite-quarkus-1.0.0-SNAPSHOT-runner.jar` file in the `/build` directory. -Be aware that it’s not an _über-jar_ as the dependencies are copied into the `build/lib` directory. + +It produces the `theodolite-quarkus-1.0.0-SNAPSHOT-runner.jar` file in the `/build` directory. Be aware that it’s not +an _über-jar_ as the dependencies are copied into the `build/lib` directory. If you want to build an _über-jar_, execute the following command: + ```shell script ./gradlew build -Dquarkus.package.type=uber-jar ``` @@ -30,11 +34,13 @@ The application is now runnable using `java -jar build/theodolite-quarkus-1.0.0- ## Creating a native executable You can create a native executable using: + ```shell script ./gradlew build -Dquarkus.package.type=native ``` Or, if you don't have GraalVM installed, you can run the native executable build in a container using: + ```shell script ./gradlew build -Dquarkus.package.type=native -Dquarkus.native.container-build=true ``` @@ -44,20 +50,73 @@ You can then execute your native executable with: If you want to learn more about building native executables, please consult https://quarkus.io/guides/gradle-tooling. -# RESTEasy JAX-RS +## Build docker images + +For the jvm version use: + +```shell script +./gradlew build +docker build -f src/main/docker/Dockerfile.jvm -t theodolite-quarkus-jvm . +``` + +For the native image version use: + +```shell script +./gradlew build -Dquarkus.package.type=native +docker build -f src/main/docker/Dockerfile.native -t theodolite-quarkus-native . +``` + +## Execute docker images: -<p>A Hello World RESTEasy resource</p> +Remember to set the environment variables first. -Guide: https://quarkus.io/guides/rest-json +Jvm version: -## Build and afterwards run the application in Docker container +```shell script +docker run -i --rm theodolite-quarkus-jvm +``` + +Native image version: -```build_jvm.sh``` to build the jvm version +```shell script +docker run -i --rm theodolite-quarkus-native +``` -```build_native.sh``` to build the native image graal version +## Environment variables -## Install Detekt Code analysis Plugin +**Production:** (Docker-Container) +| Variables name | Default value |Usage | +| -----------------------------|:----------------------------------:| ------------:| +| `NAMESPACE` | `default` |Determines the namespace of the Theodolite will be executed in. Used in the KubernetesBenchmark| +| `THEODOLITE_EXECUTION` | `./config/BenchmarkExecution.yaml`|The complete path to the benchmarkExecution file. Used in the TheodoliteYamlExecutor. | +| `THEODOLITE_BENCHMARK_TYPE` | `./config/BenchmarkType.yaml` |The complete path to the benchmarkType file. Used in the TheodoliteYamlExecutor.| +| `THEODOLITE_APP_RESOURCES` | `./config` |The path under which the yamls for the resources for the subexperiments are found. Used in the KubernetesBenchmark| +| `MODE` | `yaml-executor` | Defines the mode of operation: either `yaml-executor` or `operator`| + +**Development:** (local via Intellij) + +When running Theodolite from within IntelliJ via +[Run Configurations](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html#gradle_run_config), set the * +Environment variables* field to: + +``` +NAMESPACE=default;THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml;THEODOLITE_APP_RESOURCES=./../../../../config;THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml;MODE=operator +``` + +Alternative: + +``` sh +export NAMESPACE=default +export THEODOLITE_BENCHMARK=./../../../../config/BenchmarkType.yaml +export THEODOLITE_APP_RESOURCES=./../../../../config +export THEODOLITE_EXECUTION=./../../../../config/BenchmarkExecution.yaml +export MODE=operator +./gradlew quarkusDev + +``` + +#### Install Detekt Code analysis Plugin Install https://plugins.jetbrains.com/plugin/10761-detekt @@ -66,11 +125,10 @@ Install https://plugins.jetbrains.com/plugin/10761-detekt - Check Enable Detekt - Specify your detekt configuration and baseline file (optional) - -> detekt issues will be annotated on-the-fly while coding **ingore Failures in build:** add ```ignoreFailures = true``` - to build.gradle detekt task +to build.gradle detekt task diff --git a/theodolite-quarkus/build.gradle b/theodolite-quarkus/build.gradle index 2f58f4c8a587a4a17cf8586e507c0079d8cb56e6..8c0a13d1f24cfde01c8604285c49d3f48fe6a3b7 100644 --- a/theodolite-quarkus/build.gradle +++ b/theodolite-quarkus/build.gradle @@ -18,16 +18,19 @@ dependencies { implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' implementation 'io.quarkus:quarkus-arc' implementation 'io.quarkus:quarkus-resteasy' - testImplementation 'io.quarkus:quarkus-junit5' - testImplementation 'io.rest-assured:rest-assured' implementation 'com.google.code.gson:gson:2.8.5' - implementation 'org.slf4j:slf4j-simple:1.7.29' implementation 'io.github.microutils:kotlin-logging:1.12.0' implementation 'io.fabric8:kubernetes-client:5.0.0-alpha-2' implementation 'io.quarkus:quarkus-kubernetes-client' implementation 'org.apache.kafka:kafka-clients:2.7.0' implementation 'khttp:khttp:1.0.0' + compile 'junit:junit:4.12' + + + testImplementation 'io.quarkus:quarkus-junit5' + testImplementation 'io.rest-assured:rest-assured' + testImplementation 'org.junit-pioneer:junit-pioneer:1.4.0' } group 'theodolite' diff --git a/theodolite-quarkus/config/README.md b/theodolite-quarkus/config/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23337d77375ebba8f624e7a11f714502fe3d5e67 --- /dev/null +++ b/theodolite-quarkus/config/README.md @@ -0,0 +1,201 @@ +## The Benchmark Object + +The *benchmark* object defines all static components of an execution of a benchmark with Theodolite. +An exapmle for a benchmark object is given in [example-benchmark-yaml-resource](example-benchmark-yaml-resource.yaml). + + +A **Benchmark** is a [*standard tool for the competitive evaluation and comparison of competing systems or components according to specific characteristics, such as performance, dependability, or security*](https://doi.org/10.1145/2668930.2688819). In Theodolite, we have [specification-based benchmarks](https://doi.org/10.1145/2668930.2688819), or at least something very close to that. That is, our benchmarks are architectural descriptions---in our case---[of typical use cases of stream processing in microservices](https://doi.org/10.1016/j.bdr.2021.100209) (e.g. our UC1). Hence, we don't really have a piece of software, which represents a benchmark. We only have implementations of benchmarks, e.g. an implementation of UC1 with Kafka Streams. For simplification, we call these *benchmark implementations* simply *benchmarks*. + +```yaml +name: String +appResource: + - String + ... +loadGenResource: + - String + ... +resourceTypes: + - typeName: String + patchers: + - type: String + resources: String + properties: + <Patcher Arguments> ... + ... +loadTypes: + - typeName: String + patchers: + - type: String + resources: String + properties: + <Patcher Arguments> ... + ... +kafkaConfig: + bootstrapServer: String + topics: + - name: String + numPartitions: UnsignedInt + replicationFactor: UnsignedInt + - name: String + removeOnly: bool + ... +``` + +The properties have the following definitions: + +* **name**: The name of the *benchmark* +* **appResource**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the system under test (SUT). +* **loadGenResources**: A list of file names that reference Kubernetes resources that are deployed on the cluster for the load generator. +* **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* object. Each resource type has the following structure: + * **typeName**: Name of the resource type. + * **patchers**: List of [patchers](#Patchers) used to scale this resource type. Each patcher has the following structure: + * **type**: Type of the [patcher](#Patchers). The concrete types can be looked up in the list of [patchers](#Patchers). + * **resources**: Specifies the Kubernetes resource to be patched. + * **properties**: *Patcher Arguments*: (Optional) Patcher specific additional arguments. +* **loadTypes**: A list of load types that can be scaled for this *benchmark*. For each load type the concrete values are defined in the *execution* object. Each load type has the following structure: + * **typeName**: Name of the load type. + * **patchers**: List of patchers used to scale * **resourceTypes**: A list of resource types that can be scaled for this *benchmark*. For each resource type the concrete values are defined in the *execution* resource object.Each resource type has the following structure: + * **typeName**: Name of the resource type. + * **patchers**: List of patchers used to scale this resource type. Each patcher has the following structure: + * **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers. + * **resources**: Specifies the Kubernetes resource to be patched. + * **properties**: *Patcher Arguments*: (Optional) Patcher specific additional arguments as Map<String, String>. +* **kafkaConfig**: Contains the Kafka configuration. + * **bootstrapServers**: The bootstrap servers connection string. + * **topics**: List of topics to be created for each [experiment](#Experiment). Alternative theodolite offers the possibility to remove certain topics after each experiment. + * **name**: The name of the topic. + * **numPartitions**: The number of partitions of the topic. + * **replicationFactor**: The replication factor of the topic. + * **removeOnly**: determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic. + + +## The Execution Object + +A benchmark can be executed for different SUTs, by different users and multiple times. We call such an execution of a benchmark simply an *execution*. The *execution* object defines all conrete values of an Execution. +An exapmle for an execution object is given in [example-execution-yaml-resource](example-benchmark-yaml-resource.yaml). + + +```yaml +name: String +benchmark: String +load: + loadType: String + loadValues: + - UnsignedInt + ... +resources: + resourceType: String + resourceValues: + - UnsignedInt + ... +slos: + - sloType: String + threshold: UnsignedInt + prometheusUrl: String + externalSloUrl: String + offset: SignedInt + warmup: UnsignedInt + ... +executions: + strategy: "LinearSearch" or "BinarySearch" + duration: UnsignedInt + repetition: UnsignedInt + restrictions: + - "LowerBound" + ... +configurationOverrides: + - patcher: + type: String + resource: String + properties: + <Patcher Arguments> ... + ... +``` + +The properties have the following definitions: + +* **name**: The name of the *execution* +* **benchmark**: The name of the *benchmark* this *execution* is referring to. +* **load**: Specifies the load values that are benchmarked. + * **loadType**: The type of the load. It must match one of the load types specified in the referenced *benchmark*. + * **loadValues**: List of load values for the specified load type. +* **resources**: Specifies the scaling resource that is benchmarked. + * **resourceType**: The type of the resource. It must match one of the resource types specified in the referenced *benchmark*. + * **resourceValues**: List of resource values for the specified resource type. +* **slos**: List of the Service Level Objective (SLO) for this *execution*. Each SLO has the following fields: + * **sloType**: The type of the SLO. It must match 'lag trend'. + * **threshold**: The threshold the SUT should meet for a sucessful experiment. + * **prometheusUrl**: Connection string for promehteus. + * **externalSloUrl**: Connection string for a external slo analysis. + * **offset**: Hours by which the start and end timestamp will be shifted (for different timezones). + * **warmup**: Seconds of time that are ignored in the analysis. +* **executions**: Defines the overall parameter for the execution. + * **strategy**: Defines the used strategy for the execution: either 'LinearSearch' or 'BinarySearch' + * **duration**: Defines the duration of each [experiment](#Experiment) in seconds. + * **repetition**: Unused. + * **restrictions**: List of restriction strategys used to delimit the search space. + **- LowerBound**: Currently only supported *restriction strategy*. +* **configurationOverrides**: List of patchers that are used to override existing configurations. + * **patcher**: Patcher used to patch a resource. Each patcher has the following structure: + * **type**: Type of the Patcher. The concrete types can be looked up in the list of patchers. + * **resources**: Specifies the Kubernetes resource to be patched. + * **properties**: *Patcher Arguments*: (Optional) Patcher specific additional arguments. + +## Patchers + +* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment. + * **type**: "ReplicaPatcher" + * **resource**: "uc1-kstreams-deployment.yaml" + +* **NumSensorsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (value + 15_000 - 1) / 15_000 + * **type**: "NumSensorsLoadGeneratorReplicaPatcher" + * **resource**: "uc1-load-generator-deployment.yaml" + +* **NumNestedGroupsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (4^(value) + 15_000 -1) /15_000 + * **type**: "NumNestedGroupsLoadGeneratorReplicaPatcher" + * **resource**: "uc1-load-generator-deployment.yaml" + +* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment. + * **type**: "ReplicaPatcher" + * **resource**: "uc1-kstreams-deployment.yaml" + +* **EnvVarPatcher**: Allows to modify the value of an environment variable for a container in a kubernetes deployment. + * **type**: "EnvVarPatcher" + * **resource**: "uc1-load-generator-deployment.yaml" + * **properties**: + * container: "workload-generator" + * variableName: "NUM_SENSORS" + +* **NodeSelectorPatcher**: Changes the node selection field in kubernetes resources. + * **type**: "NodeSelectorPatcher" + * **resource**: "uc1-load-generator-deployment.yaml" + * **properties**: + * variableName: "env" + * **value**: "prod" + +* **ResourceLimitPatcher**: Changes the resource limit for a kubernetes resource. + * **resource**: "uc1-kstreams-deployment.yaml" + * **properties**: + * container: "uc-application" + * variableName: "cpu" or "memory" + * **value**:"1000m" or "2Gi" + +* **SchedulerNamePatcher**: Changes the sheduler for kubernetes resources. + * **type**: "SchedulerNamePatcher" + * **resource**: "uc1-kstreams-deployment.yaml" + * **value**: "random-scheduler" + +* **ImagePatcher**: Changes the image of a kubernetes resource. Currently not fully implemented. + * **type**: "ImagePatcher" + * **resource**: "uc1-kstreams-deployment.yaml" + * **properties**: + * container: "uc-application" + * **value**: "dockerhubrepo/imagename" + + + +## Experiment +According to [our benchmarking method](https://doi.org/10.1016/j.bdr.2021.100209), the execution of a benchmark requires performing multiple **Experiments**. I think what is actually done within/during an experiment is another level of detail. (But just for the sake of completeness: In an experiment, the benchmark implementation is deployed, load is generated according to the benchmark specification, some SLOs are monitored continuously, etc.) + + + diff --git a/theodolite-quarkus/config/example-execution-yaml-resource.yaml b/theodolite-quarkus/config/example-execution-yaml-resource.yaml index 23c1587ec1e5c2a88fcf69d7127edbcc1ffdb00f..e46a6cf417442b851650fe9699f73e1367dcc794 100644 --- a/theodolite-quarkus/config/example-execution-yaml-resource.yaml +++ b/theodolite-quarkus/config/example-execution-yaml-resource.yaml @@ -17,31 +17,36 @@ execution: strategy: "LinearSearch" duration: 300 # in seconds repetitions: 1 + loadGenerationDelay: 30 # in seconds, optional field, default is 0 seconds restrictions: - "LowerBound" -configOverrides: [] -# - patcher: -# type: "NodeSelectorPatcher" -# resource: "uc1-load-generator-deployment.yaml" -# variableName: "env" -# value: "prod" -# - patcher: -# type: "NodeSelectorPatcher" -# resource: "uc1-kstreams-deployment.yaml" -# variableName: "env" -# value: "prod" -# - patcher: -# type: "ResourceLimitPatcher" -# resource: "uc1-kstreams-deployment.yaml" -# container: "uc-application" -# variableName: "cpu" -# value: "1000m" -# - patcher: -# type: "ResourceLimitPatcher" -# resource: "uc1-kstreams-deployment.yaml" -# container: "uc-application" -# variableName: "memory" -# value: "2Gi" +configOverrides: + - patcher: + type: "NodeSelectorPatcher" + resource: "uc1-load-generator-deployment.yaml" + properties: + variableName: "env" + value: "prod" + - patcher: + type: "NodeSelectorPatcher" + resource: "uc1-kstreams-deployment.yaml" + properties: + variableName: "env" + value: "prod" + - patcher: + type: "ResourceLimitPatcher" + resource: "uc1-kstreams-deployment.yaml" + properties: + container: "uc-application" + limitedResource: "cpu" + value: "1000m" + - patcher: + type: "ResourceLimitPatcher" + resource: "uc1-kstreams-deployment.yaml" + properties: + container: "uc-application" + limitedResource: "memory" + value: "2Gi" # - patcher: # type: "SchedulerNamePatcher" # resource: "uc1-kstreams-deployment.yaml" diff --git a/theodolite-quarkus/config/example-operator-benchmark.yaml b/theodolite-quarkus/config/example-operator-benchmark.yaml deleted file mode 100644 index 419042fdd0b1e58fed4d402b4bb329d54602d23f..0000000000000000000000000000000000000000 --- a/theodolite-quarkus/config/example-operator-benchmark.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: theodolite.com/v1alpha1 -kind: benchmark -metadata: - name: uc1-kstreams -#name: "uc1-kstreams" -appResource: - - "uc1-kstreams-deployment.yaml" - - "aggregation-service.yaml" - - "jmx-configmap.yaml" - - "uc1-service-monitor.yaml" -loadGenResource: - - "uc1-load-generator-deployment.yaml" - - "uc1-load-generator-service.yaml" -resourceTypes: - - typeName: "Instances" - patchers: - - type: "ReplicaPatcher" - resource: "uc1-kstreams-deployment.yaml" -loadTypes: - - typeName: "NumSensors" - patchers: - - type: "EnvVarPatcher" - resource: "uc1-load-generator-deployment.yaml" - container: "workload-generator" - variableName: "NUM_SENSORS" -kafkaConfig: - bootstrapServer: "theodolite-cp-kafka:9092" - topics: - - name: "input" - numPartitions: 40 - replicationFactor: 1 \ No newline at end of file diff --git a/theodolite-quarkus/config/example-operator-execution.yaml b/theodolite-quarkus/config/example-operator-execution.yaml index 3df1a723dd771453ab1b267335176e4ae74c3ed5..9b2a1facbd8be3411407dfcf3cad39fd9f3de6b6 100644 --- a/theodolite-quarkus/config/example-operator-execution.yaml +++ b/theodolite-quarkus/config/example-operator-execution.yaml @@ -1,52 +1,57 @@ -apiVersion: theodolite.com/v1alpha1 +apiVersion: theodolite.com/v1 kind: execution metadata: name: example-execution -#name: example-execution -benchmark: "uc1-kstreams" -load: - loadType: "NumSensors" - loadValues: [25000, 50000, 75000, 100000, 125000, 150000] -resources: - resourceType: "Instances" - resourceValues: [1, 2, 3, 4, 5] -slos: - - sloType: "lag trend" - threshold: 2000 - prometheusUrl: "http://prometheus-operated:9090" - externalSloUrl: "http://localhost:80/evaluate-slope" - offset: 0 - warmup: 60 # in seconds -execution: - strategy: "LinearSearch" - duration: 300 # in seconds - repetitions: 1 - restrictions: - - "LowerBound" -configOverrides: [] -# - patcher: -# type: "NodeSelectorPatcher" -# resource: "uc1-load-generator-deployment.yaml" -# variableName: "env" -# value: "prod" -# - patcher: -# type: "NodeSelectorPatcher" -# resource: "uc1-kstreams-deployment.yaml" -# variableName: "env" -# value: "prod" -# - patcher: -# type: "ResourceLimitPatcher" -# resource: "uc1-kstreams-deployment.yaml" -# container: "uc-application" -# variableName: "cpu" -# value: "1000m" -# - patcher: -# type: "ResourceLimitPatcher" -# resource: "uc1-kstreams-deployment.yaml" -# container: "uc-application" -# variableName: "memory" -# value: "2Gi" -# - patcher: -# type: "SchedulerNamePatcher" -# resource: "uc1-kstreams-deployment.yaml" -# value: "random-scheduler" +spec: + benchmark: "uc1-kstreams" + load: + loadType: "NumSensors" + loadValues: [25000, 50000, 75000, 100000, 125000, 150000] + resources: + resourceType: "Instances" + resourceValues: [1, 2, 3, 4, 5] + slos: + - sloType: "lag trend" + threshold: 2000 + prometheusUrl: "http://prometheus-operated:9090" + externalSloUrl: "http://localhost:80/evaluate-slope" + offset: 0 + warmup: 60 # in seconds + execution: + strategy: "LinearSearch" + duration: 300 # in seconds + repetitions: 1 + loadGenerationDelay: 30 # in seconds + restrictions: + - "LowerBound" + configOverrides: + # - patcher: + # type: "NodeSelectorPatcher" + # resource: "uc1-load-generator-deployment.yaml" + # properties: + # variableName: "env" + # value: "prod" + # - patcher: + # type: "NodeSelectorPatcher" + # resource: "uc1-kstreams-deployment.yaml" + # properties: + # variableName: "env" + # value: "prod" + # - patcher: + # type: "ResourceLimitPatcher" + # resource: "uc1-kstreams-deployment.yaml" + # properties: + # container: "uc-application" + # limitedResource: "cpu" + # value: "1000m" + # - patcher: + # type: "ResourceLimitPatcher" + # resource: "uc1-kstreams-deployment.yaml" + # properties: + # container: "uc-application" + # limitedResource: "memory" + # value: "2Gi" + # - patcher: + # type: "SchedulerNamePatcher" + # resource: "uc1-kstreams-deployment.yaml" + # value: "random-scheduler" diff --git a/theodolite-quarkus/config/thedolite-operator.yaml b/theodolite-quarkus/config/thedolite-operator.yaml deleted file mode 100644 index 1e0e60248c2474cc8493179c003b806030f79f8c..0000000000000000000000000000000000000000 --- a/theodolite-quarkus/config/thedolite-operator.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: theodolite -spec: - selector: - matchLabels: - app: theodolite - replicas: 1 - template: - metadata: - labels: - app: theodolite - spec: - terminationGracePeriodSeconds: 0 - serviceAccountName: theodolite - containers: - - name: thedolite - image: ghcr.io/cau-se/theodolite:theodolite-kotlin-latest - env: - - name: KUBECONFIG - value: "~/.kube/config" - - name: NAMESPACE - value: "default" \ No newline at end of file diff --git a/theodolite-quarkus/crd/crd-benchmark.yaml b/theodolite-quarkus/crd/crd-benchmark.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b76821f6e7cca5408f604ba9bbf83cf1b43a37de --- /dev/null +++ b/theodolite-quarkus/crd/crd-benchmark.yaml @@ -0,0 +1,117 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: benchmarks.theodolite.com +spec: + group: theodolite.com + names: + kind: benchmark + plural: benchmarks + shortNames: + - bench + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + required: ["spec"] + properties: + spec: + type: object + required: [] + properties: + name: + type: string + appResource: + type: array + minItems: 1 + items: + type: string + loadGenResource: + type: array + minItems: 1 + items: + type: string + resourceTypes: + type: array + minItems: 1 + items: + type: object + properties: + typeName: + type: string + patchers: + type: array + minItems: 1 + items: + type: object + properties: + type: + type: string + default: "" + resource: + type: string + default: "" + properties: + type: object + additionalProperties: true + x-kubernetes-map-type: "granular" + default: {} + loadTypes: + type: array + minItems: 1 + items: + type: object + properties: + typeName: + type: string + patchers: + type: array + minItems: 1 + items: + type: object + properties: + type: + type: string + default: "" + resource: + type: string + default: "" + properties: + type: object + additionalProperties: true + x-kubernetes-map-type: "granular" + default: {} + kafkaConfig: + type: object + properties: + bootstrapServer: + type: string + topics: + type: array + minItems: 1 + items: + type: object + required: [] + properties: + name: + type: string + default: "" + numPartitions: + type: integer + default: 0 + replicationFactor: + type: integer + default: 0 + removeOnly: + type: boolean + default: false + additionalPrinterColumns: + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + scope: Namespaced \ No newline at end of file diff --git a/theodolite-quarkus/crd/crd-execution.yaml b/theodolite-quarkus/crd/crd-execution.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b984f3ebe5ca7c8868adb9d3593e5d87d73fc2bd --- /dev/null +++ b/theodolite-quarkus/crd/crd-execution.yaml @@ -0,0 +1,128 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: executions.theodolite.com +spec: + group: theodolite.com + names: + kind: execution + plural: executions + shortNames: + - exec + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + required: ["spec"] + properties: + spec: + type: object + required: ["benchmark", "load", "resources", "slos", "execution", "configOverrides"] + properties: + name: + type: string + default: "" + benchmark: + type: string + load: # definition of the load dimension + type: object + required: ["loadType", "loadValues"] + properties: + loadType: + type: string + loadValues: + type: array + items: + type: integer + resources: # definition of the resource dimension + type: object + required: ["resourceType", "resourceValues"] + properties: + resourceType: + type: string + resourceValues: + type: array + items: + type: integer + slos: # def of service level objectives + type: array + items: + type: object + required: ["sloType", "threshold", "prometheusUrl", "externalSloUrl", "offset", "warmup"] + properties: + sloType: + type: string + threshold: + type: integer + prometheusUrl: + type: string + externalSloUrl: + type: string + offset: + type: integer + warmup: + type: integer + execution: # def execution config + type: object + required: ["strategy", "duration", "repetitions", "restrictions"] + properties: + strategy: + type: string + duration: + type: integer + repetitions: + type: integer + loadGenerationDelay: + type: integer + restrictions: + type: array + items: + type: string + configOverrides: + type: array + items: + type: object + properties: + patcher: + type: object + properties: + type: + type: string + default: "" + resource: + type: string + default: "" + properties: + type: object + additionalProperties: true + x-kubernetes-map-type: "granular" + default: {} + value: + type: string + status: + type: object + properties: + executionState: + description: "" + type: string + executionDuration: + description: "Duration of the execution in seconds" + type: string + additionalPrinterColumns: + - name: STATUS + type: string + description: State of the execution + jsonPath: .status.executionState + - name: Duration + type: string + description: Duration of the execution + jsonPath: .status.executionDuration + - name: Age + type: date + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + scope: Namespaced \ No newline at end of file diff --git a/theodolite-quarkus/examples/operator/example-benchmark.yaml b/theodolite-quarkus/examples/operator/example-benchmark.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91d9f8f1f7dfed31d9edcb59947af4e832ca2843 --- /dev/null +++ b/theodolite-quarkus/examples/operator/example-benchmark.yaml @@ -0,0 +1,38 @@ +apiVersion: theodolite.com/v1 +kind: benchmark +metadata: + name: uc1-kstreams +spec: + appResource: + - "uc1-kstreams-deployment.yaml" + - "aggregation-service.yaml" + - "jmx-configmap.yaml" + - "uc1-service-monitor.yaml" + loadGenResource: + - "uc1-load-generator-deployment.yaml" + - "uc1-load-generator-service.yaml" + resourceTypes: + - typeName: "Instances" + patchers: + - type: "ReplicaPatcher" + resource: "uc1-kstreams-deployment.yaml" + loadTypes: + - typeName: "NumSensors" + patchers: + - type: "EnvVarPatcher" + resource: "uc1-load-generator-deployment.yaml" + properties: + variableName: "NUM_SENSORS" + container: "workload-generator" + - type: "NumSensorsLoadGeneratorReplicaPatcher" + resource: "uc1-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" + kafkaConfig: + bootstrapServer: "theodolite-cp-kafka:9092" + topics: + - name: "input" + numPartitions: 40 + replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True \ No newline at end of file diff --git a/theodolite-quarkus/examples/operator/example-execution.yaml b/theodolite-quarkus/examples/operator/example-execution.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5386fd7c8665e01302067da81c5dd4caf87fc602 --- /dev/null +++ b/theodolite-quarkus/examples/operator/example-execution.yaml @@ -0,0 +1,57 @@ +apiVersion: theodolite.com/v1 +kind: execution +metadata: + name: theodolite-example-execution +spec: + benchmark: "uc1-kstreams" + load: + loadType: "NumSensors" + loadValues: [25000, 50000, 75000, 100000, 125000, 150000] + resources: + resourceType: "Instances" + resourceValues: [1, 2, 3, 4, 5] + slos: + - sloType: "lag trend" + threshold: 2000 + prometheusUrl: "http://prometheus-operated:9090" + externalSloUrl: "http://localhost:80/evaluate-slope" + offset: 0 + warmup: 60 # in seconds + execution: + strategy: "LinearSearch" + duration: 300 # in seconds + repetitions: 1 + loadGenerationDelay: 30 # in seconds + restrictions: + - "LowerBound" + configOverrides: + # - patcher: + # type: "NodeSelectorPatcher" + # resource: "uc1-load-generator-deployment.yaml" + # properties: + # variableName: "env" + # value: "prod" + # - patcher: + # type: "NodeSelectorPatcher" + # resource: "uc1-kstreams-deployment.yaml" + # properties: + # variableName: "env" + # value: "prod" + # - patcher: + # type: "ResourceLimitPatcher" + # resource: "uc1-kstreams-deployment.yaml" + # properties: + # container: "uc-application" + # limitedResource: "cpu" + # value: "1000m" + # - patcher: + # type: "ResourceLimitPatcher" + # resource: "uc1-kstreams-deployment.yaml" + # properties: + # container: "uc-application" + # limitedResource: "memory" + # value: "2Gi" + # - patcher: + # type: "SchedulerNamePatcher" + # resource: "uc1-kstreams-deployment.yaml" + # value: "random-scheduler" diff --git a/theodolite-quarkus/src/main/resources/operator/example-benchmark-k8s-resource.yaml b/theodolite-quarkus/examples/standalone/example-benchmark.yaml similarity index 59% rename from theodolite-quarkus/src/main/resources/operator/example-benchmark-k8s-resource.yaml rename to theodolite-quarkus/examples/standalone/example-benchmark.yaml index 9fc16f92e303f05a449f7e8b83600c3b299f215d..83edce93834ca9b8eef5606c1e5884ce40bdd7d8 100644 --- a/theodolite-quarkus/src/main/resources/operator/example-benchmark-k8s-resource.yaml +++ b/theodolite-quarkus/examples/standalone/example-benchmark.yaml @@ -1,11 +1,9 @@ -apiVersion: theodolite.com/v1alpha1 -kind: benchmark -metadata: - name: uc1-kstreams +name: "uc1-kstreams" appResource: - "uc1-kstreams-deployment.yaml" - "aggregation-service.yaml" - "jmx-configmap.yaml" + - "uc1-service-monitor.yaml" loadGenResource: - "uc1-load-generator-deployment.yaml" - "uc1-load-generator-service.yaml" @@ -19,11 +17,18 @@ loadTypes: patchers: - type: "EnvVarPatcher" resource: "uc1-load-generator-deployment.yaml" - container: "workload-generator" - variableName: "NUM_SENSORS" + properties: + variableName: "NUM_SENSORS" + container: "workload-generator" + - type: "NumSensorsLoadGeneratorReplicaPatcher" + resource: "uc1-load-generator-deployment.yaml" + properties: + loadGenMaxRecords: "15000" kafkaConfig: bootstrapServer: "localhost:31290" topics: - name: "input" numPartitions: 40 - replicationFactor: 1 \ No newline at end of file + replicationFactor: 1 + - name: "theodolite-.*" + removeOnly: True diff --git a/theodolite-quarkus/examples/standalone/example-execution.yaml b/theodolite-quarkus/examples/standalone/example-execution.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24b2b7f32e803553a4a13b76869ccf4cf3f6e5a5 --- /dev/null +++ b/theodolite-quarkus/examples/standalone/example-execution.yaml @@ -0,0 +1,23 @@ +name: example-execution +benchmark: "uc1-kstreams" +load: + loadType: "NumSensors" + loadValues: [25000, 50000, 75000, 100000, 125000, 150000] +resources: + resourceType: "Instances" + resourceValues: [1, 2, 3, 4, 5] +slos: + - sloType: "lag trend" + threshold: 2000 + prometheusUrl: "http://prometheus-operated:9090" + externalSloUrl: "http://localhost:80/evaluate-slope" + offset: 0 + warmup: 60 # in seconds +execution: + strategy: "LinearSearch" + duration: 300 # in seconds + repetitions: 1 + loadGenerationDelay: 30 # in seconds, optional field, default is 0 seconds + restrictions: + - "LowerBound" +configOverrides: [] \ No newline at end of file diff --git a/theodolite-quarkus/src/main/docker/Dockerfile.native b/theodolite-quarkus/src/main/docker/Dockerfile.native index 37a5a4bd8472a358194dbb14a5fce61df94804d3..29836a7148b573c3051c33341718b06008fa07e2 100644 --- a/theodolite-quarkus/src/main/docker/Dockerfile.native +++ b/theodolite-quarkus/src/main/docker/Dockerfile.native @@ -15,12 +15,12 @@ # ### FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3 -WORKDIR /work/ -RUN chown 1001 /work \ - && chmod "g+rwX" /work \ - && chown 1001:root /work -COPY --chown=1001:root build/*-runner /work/application -COPY config/ /work/config/ +WORKDIR /deployments +RUN chown 1001 /deployments \ + && chmod "g+rwX" /deployments \ + && chown 1001:root /deployments +COPY --chown=1001:root build/*-runner /deployments/application +COPY config/ /deployments/config/ EXPOSE 8080 USER 1001 diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt index db7999b205c61d94fa17791a5d549a2620601b6b..d57a28e8bbcf4dc101e4814ecaa0d52fe28c08a9 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/Benchmark.kt @@ -1,5 +1,8 @@ package theodolite.benchmark +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.api.model.Namespaced +import io.fabric8.kubernetes.client.CustomResource import io.quarkus.runtime.annotations.RegisterForReflection import theodolite.util.ConfigurationOverride import theodolite.util.LoadDimension @@ -21,6 +24,8 @@ interface Benchmark { fun buildDeployment( load: LoadDimension, res: Resource, - configurationOverrides: List<ConfigurationOverride?> + configurationOverrides: List<ConfigurationOverride?>, + loadGenerationDelay: Long, + afterTeardownDelay: Long ): BenchmarkDeployment } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt index 2d5d15b3389cf723be3a8ceb0fff8b27bd700419..62ab75898d16ff2732ab6aa5c254ec8f87fb7266 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecution.kt @@ -2,8 +2,6 @@ package theodolite.benchmark import com.fasterxml.jackson.databind.annotation.JsonDeserialize import io.fabric8.kubernetes.api.model.KubernetesResource -import io.fabric8.kubernetes.api.model.Namespaced -import io.fabric8.kubernetes.client.CustomResource import io.quarkus.runtime.annotations.RegisterForReflection import theodolite.util.ConfigurationOverride import kotlin.properties.Delegates @@ -26,7 +24,7 @@ import kotlin.properties.Delegates */ @JsonDeserialize @RegisterForReflection -class BenchmarkExecution : CustomResource(), Namespaced { +class BenchmarkExecution : KubernetesResource { var executionId: Int = 0 lateinit var name: String lateinit var benchmark: String @@ -34,7 +32,7 @@ class BenchmarkExecution : CustomResource(), Namespaced { lateinit var resources: ResourceDefinition lateinit var slos: List<Slo> lateinit var execution: Execution - lateinit var configOverrides: List<ConfigurationOverride?> + lateinit var configOverrides: MutableList<ConfigurationOverride?> /** * This execution encapsulates the [strategy], the [duration], the [repetitions], and the [restrictions] @@ -47,6 +45,8 @@ class BenchmarkExecution : CustomResource(), Namespaced { var duration by Delegates.notNull<Long>() var repetitions by Delegates.notNull<Int>() lateinit var restrictions: List<String> + var loadGenerationDelay = 0L + var afterTeardownDelay = 5L } /** diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecutionList.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecutionList.kt deleted file mode 100644 index 50e8967f20aebad880ebd218136749af8e3ea6ee..0000000000000000000000000000000000000000 --- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/BenchmarkExecutionList.kt +++ /dev/null @@ -1,5 +0,0 @@ -package theodolite.benchmark - -import io.fabric8.kubernetes.client.CustomResourceList - -class BenchmarkExecutionList : CustomResourceList<BenchmarkExecution>() \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt index 71b65a28fd074e4554c283ee94a8db028d652d46..aa9c36ad912437e3b104dccf6ff1f4dea5905946 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt @@ -1,5 +1,6 @@ package theodolite.benchmark +import com.fasterxml.jackson.databind.annotation.JsonDeserialize import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.api.model.Namespaced import io.fabric8.kubernetes.client.CustomResource @@ -22,7 +23,7 @@ private var DEFAULT_NAMESPACE = "default" * - [loadGenResource] resource that generates the load, * - [resourceTypes] types of scaling resources, * - [loadTypes] types of loads that can be scaled for the benchmark, - * - [kafkaConfig] for the [TopicManager], + * - [kafkaConfig] for the [theodolite.k8s.TopicManager], * - [namespace] for the client, * - [path] under which the resource yamls can be found. * @@ -30,16 +31,18 @@ private var DEFAULT_NAMESPACE = "default" * for the deserializing in the [theodolite.execution.operator.TheodoliteOperator]. * @constructor construct an empty Benchmark. */ +@JsonDeserialize @RegisterForReflection -class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced { +class KubernetesBenchmark: KubernetesResource, Benchmark{ lateinit var name: String lateinit var appResource: List<String> lateinit var loadGenResource: List<String> lateinit var resourceTypes: List<TypeName> lateinit var loadTypes: List<TypeName> lateinit var kafkaConfig: KafkaConfig - private val namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE - var path = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config" + var namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE + var path = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config" + /** * Loads [KubernetesResource]s. @@ -63,40 +66,47 @@ class KubernetesBenchmark : Benchmark, CustomResource(), Namespaced { * First loads all required resources and then patches them to the concrete load and resources for the experiment. * Afterwards patches additional configurations(cluster depending) into the resources. * @param load concrete load that will be benchmarked in this experiment. - * @param res concrete resoruce that will be scaled for this experiment. + * @param res concrete resource that will be scaled for this experiment. * @param configurationOverrides * @return a [BenchmarkDeployment] */ override fun buildDeployment( load: LoadDimension, res: Resource, - configurationOverrides: List<ConfigurationOverride?> + configurationOverrides: List<ConfigurationOverride?>, + loadGenerationDelay: Long, + afterTeardownDelay: Long ): BenchmarkDeployment { logger.info { "Using $namespace as namespace." } logger.info { "Using $path as resource path." } - val resources = loadKubernetesResources(this.appResource + this.loadGenResource) + val appResources = loadKubernetesResources(this.appResource) + val loadGenResources = loadKubernetesResources(this.loadGenResource) + val patcherFactory = PatcherFactory() // patch the load dimension the resources load.getType().forEach { patcherDefinition -> - patcherFactory.createPatcher(patcherDefinition, resources).patch(load.get().toString()) + patcherFactory.createPatcher(patcherDefinition, loadGenResources).patch(load.get().toString()) } res.getType().forEach { patcherDefinition -> - patcherFactory.createPatcher(patcherDefinition, resources).patch(res.get().toString()) + patcherFactory.createPatcher(patcherDefinition, appResources).patch(res.get().toString()) } // Patch the given overrides configurationOverrides.forEach { override -> override?.let { - patcherFactory.createPatcher(it.patcher, resources).patch(override.value) + patcherFactory.createPatcher(it.patcher, appResources + loadGenResources).patch(override.value) } } return KubernetesBenchmarkDeployment( namespace = namespace, - resources = resources.map { r -> r.second }, + appResources = appResources.map { it.second }, + loadGenResources = loadGenResources.map { it.second }, + loadGenerationDelay = loadGenerationDelay, + afterTeardownDelay = afterTeardownDelay, kafkaConfig = hashMapOf("bootstrap.servers" to kafkaConfig.bootstrapServer), - topics = kafkaConfig.getKafkaTopics(), + topics = kafkaConfig.topics, client = DefaultKubernetesClient().inNamespace(namespace) ) } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt index 0de00fb5007b646240661310d037e34ef1ea1ae2..6cf239676ddb24752f4754a85fc62657f9eb6603 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt @@ -3,9 +3,14 @@ package theodolite.benchmark import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.quarkus.runtime.annotations.RegisterForReflection +import mu.KotlinLogging import org.apache.kafka.clients.admin.NewTopic import theodolite.k8s.K8sManager import theodolite.k8s.TopicManager +import theodolite.util.KafkaConfig +import java.time.Duration + +private val logger = KotlinLogging.logger {} /** * Organizes the deployment of benchmarks in Kubernetes. @@ -18,14 +23,17 @@ import theodolite.k8s.TopicManager @RegisterForReflection class KubernetesBenchmarkDeployment( val namespace: String, - val resources: List<KubernetesResource>, + val appResources: List<KubernetesResource>, + val loadGenResources: List<KubernetesResource>, + private val loadGenerationDelay: Long, + private val afterTeardownDelay: Long, private val kafkaConfig: HashMap<String, Any>, - private val topics: Collection<NewTopic>, + private val topics: List<KafkaConfig.TopicWrapper>, private val client: NamespacedKubernetesClient ) : BenchmarkDeployment { private val kafkaController = TopicManager(this.kafkaConfig) private val kubernetesManager = K8sManager(client) - private val LABEL = "app.kubernetes.io/name=kafka-lag-exporter" + private val LAG_EXPORTER_POD_LABEL = "app.kubernetes.io/name=kafka-lag-exporter" /** * Setup a [KubernetesBenchmark] using the [TopicManager] and the [K8sManager]: @@ -33,10 +41,13 @@ class KubernetesBenchmarkDeployment( * - Deploy the needed resources. */ override fun setup() { - kafkaController.createTopics(this.topics) - resources.forEach { - kubernetesManager.deploy(it) - } + val kafkaTopics = this.topics.filter { !it.removeOnly } + .map { NewTopic(it.name, it.numPartitions, it.replicationFactor) } + kafkaController.createTopics(kafkaTopics) + appResources.forEach { kubernetesManager.deploy(it) } + logger.info { "Wait ${this.loadGenerationDelay} seconds before starting the load generator." } + Thread.sleep(Duration.ofSeconds(this.loadGenerationDelay).toMillis()) + loadGenResources.forEach { kubernetesManager.deploy(it) } } /** @@ -46,10 +57,11 @@ class KubernetesBenchmarkDeployment( * - Remove the [KubernetesResource]s. */ override fun teardown() { - resources.forEach { - kubernetesManager.remove(it) - } - kafkaController.removeTopics(this.topics.map { topic -> topic.name() }) - KafkaLagExporterRemover(client).remove(LABEL) + loadGenResources.forEach { kubernetesManager.remove(it) } + appResources.forEach { kubernetesManager.remove(it) } + kafkaController.removeTopics(this.topics.map { topic -> topic.name }) + KafkaLagExporterRemover(client).remove(LAG_EXPORTER_POD_LABEL) + logger.info { "Teardown complete. Wait $afterTeardownDelay ms to let everything come down." } + Thread.sleep(Duration.ofSeconds(afterTeardownDelay).toMillis()) } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkList.kt b/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkList.kt deleted file mode 100644 index 0930875e96146fda58301478bda68b00c229e99f..0000000000000000000000000000000000000000 --- a/theodolite-quarkus/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkList.kt +++ /dev/null @@ -1,5 +0,0 @@ -package theodolite.benchmark - -import io.fabric8.kubernetes.client.CustomResourceList - -class KubernetesBenchmarkList : CustomResourceList<KubernetesBenchmark>() \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt index e76f6cf34fdec447fa4d581b94bbb51b972d888a..ef4d371173c7099eb091f90cddbe26d31e6522be 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt @@ -2,10 +2,14 @@ package theodolite.evaluation import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution +import theodolite.util.IOHandler import theodolite.util.LoadDimension import theodolite.util.Resource +import java.text.Normalizer import java.time.Duration import java.time.Instant +import java.util.* +import java.util.regex.Pattern private val logger = KotlinLogging.logger {} @@ -31,17 +35,28 @@ class AnalysisExecutor( * @param executionDuration of the experiment. * @return true if the experiment succeeded. */ - fun analyze(load: LoadDimension, res: Resource, executionDuration: Duration): Boolean { + fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean { var result = false + var repetitionCounter = 1 try { - val prometheusData = fetcher.fetchMetric( - start = Instant.now().minus(executionDuration), - end = Instant.now(), - query = "sum by(group)(kafka_consumergroup_group_lag >= 0)" - ) + val ioHandler = IOHandler() + val resultsFolder: String = ioHandler.getResultFolderURL() + val fileURL = "${resultsFolder}exp${executionId}_${load.get()}_${res.get()}_${slo.sloType.toSlug()}" + + val prometheusData = executionIntervals + .map { interval -> fetcher.fetchMetric( + start = interval.first, + end = interval.second, + query = "sum by(group)(kafka_consumergroup_group_lag >= 0)") } + + prometheusData.forEach{ data -> + ioHandler.writeToCSVFile( + fileURL = "${fileURL}_${repetitionCounter++}", + data = data.getResultAsList(), + columns = listOf("group", "timestamp", "value")) + } - CsvExporter().toCsv(name = "$executionId-${load.get()}-${res.get()}-${slo.sloType}", prom = prometheusData) val sloChecker = SloCheckerFactory().create( sloType = slo.sloType, externalSlopeURL = slo.externalSloUrl, @@ -49,14 +64,21 @@ class AnalysisExecutor( warmup = slo.warmup ) - result = sloChecker.evaluate( - start = Instant.now().minus(executionDuration), - end = Instant.now(), fetchedData = prometheusData - ) + result = sloChecker.evaluate(prometheusData) } catch (e: Exception) { - logger.error { "Evaluation failed for resource: ${res.get()} and load: ${load.get()} error: $e" } + logger.error { "Evaluation failed for resource '${res.get()}' and load '${load.get()}'. Error: $e" } } return result } + + private val NONLATIN: Pattern = Pattern.compile("[^\\w-]") + private val WHITESPACE: Pattern = Pattern.compile("[\\s]") + + fun String.toSlug(): String { + val noWhitespace: String = WHITESPACE.matcher(this).replaceAll("-") + val normalized: String = Normalizer.normalize(noWhitespace, Normalizer.Form.NFD) + val slug: String = NONLATIN.matcher(normalized).replaceAll("") + return slug.toLowerCase(Locale.ENGLISH) + } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt deleted file mode 100644 index 68862851523934c533cf3af41f0a786ba2b5a73f..0000000000000000000000000000000000000000 --- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/CsvExporter.kt +++ /dev/null @@ -1,51 +0,0 @@ -package theodolite.evaluation - -import mu.KotlinLogging -import theodolite.util.PrometheusResponse -import java.io.File -import java.io.PrintWriter -import java.util.* - -private val logger = KotlinLogging.logger {} - -/** - * Used to document the data received from prometheus for additional offline analysis. - */ -class CsvExporter { - - /** - * Uses the [PrintWriter] to transform a [PrometheusResponse] to a CSV file. - * @param name of the file. - * @param prom Response that is documented. - * - */ - fun toCsv(name: String, prom: PrometheusResponse) { - val responseArray = promResponseToList(prom) - val csvOutputFile = File("$name.csv") - - PrintWriter(csvOutputFile).use { pw -> - pw.println(listOf("name", "time", "value").joinToString()) - responseArray.forEach { - pw.println(it.joinToString()) - } - } - logger.info { "Wrote csv file: $name to ${csvOutputFile.absolutePath}" } - } - - /** - * Converts a [PrometheusResponse] into a [List] of [List]s of [String]s - */ - private fun promResponseToList(prom: PrometheusResponse): List<List<String>> { - val name = prom.data?.result?.get(0)?.metric?.group.toString() - val values = prom.data?.result?.get(0)?.values - val dataList = mutableListOf<List<String>>() - - if (values != null) { - for (x in values) { - val y = x as List<*> - dataList.add(listOf(name, "${y[0]}", "${y[1]}")) - } - } - return Collections.unmodifiableList(dataList) - } -} diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt index fd901abc470a1f4b739b26e30276366e6bc69739..f7ebee8faf740583dbe6a37381a599e9bde19280 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/ExternalSloChecker.kt @@ -17,8 +17,7 @@ class ExternalSloChecker( private val externalSlopeURL: String, private val threshold: Int, private val warmup: Int -) : - SloChecker { +) : SloChecker { private val RETRIES = 2 private val TIMEOUT = 60.0 @@ -36,21 +35,25 @@ class ExternalSloChecker( * @return true if the experiment was successful(the threshold was not exceeded. * @throws ConnectException if the external service could not be reached. */ - override fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean { + override fun evaluate(fetchedData: List<PrometheusResponse>): Boolean { var counter = 0 - val data = - Gson().toJson(mapOf("total_lag" to fetchedData.data?.result, "threshold" to threshold, "warmup" to warmup)) + val data = Gson().toJson(mapOf( + "total_lags" to fetchedData.map { it.data?.result}, + "threshold" to threshold, + "warmup" to warmup)) while (counter < RETRIES) { val result = post(externalSlopeURL, data = data, timeout = TIMEOUT) if (result.statusCode != 200) { counter++ - logger.error { "Could not reach external slope analysis" } + logger.error { "Could not reach external SLO checker" } } else { - return result.text.toBoolean() + val booleanResult = result.text.toBoolean() + logger.info { "SLO checker result is: $booleanResult" } + return booleanResult } } - throw ConnectException("Could not reach slope evaluation") + throw ConnectException("Could not reach external SLO checker") } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt index bbfbf8c3269e442188f92a9b057fcc264acbbe78..833d7d1e16c2fbc91b58817b319a7d02af7f5b2b 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt @@ -48,18 +48,18 @@ class MetricFetcher(private val prometheusURL: String, private val offset: Durat val response = get("$prometheusURL/api/v1/query_range", params = parameter, timeout = TIMEOUT) if (response.statusCode != 200) { val message = response.jsonObject.toString() - logger.warn { "Could not connect to Prometheus: $message, retrying now" } + logger.warn { "Could not connect to Prometheus: $message. Retrying now." } counter++ } else { val values = parseValues(response) if (values.data?.result.isNullOrEmpty()) { - logger.error { "Empty query result: $values between $start and $end for querry $query" } + logger.error { "Empty query result: $values between $start and $end for query $query." } throw NoSuchFieldException() } return parseValues(response) } } - throw ConnectException("No answer from Prometheus received") + throw ConnectException("No answer from Prometheus received.") } /** diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt index 94d816d87923f4d8343c6c83dd9747f1cc25ff81..9ee5fe7ef34ce5b6214882ce2c1d19677f1d7130 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/evaluation/SloChecker.kt @@ -1,14 +1,12 @@ package theodolite.evaluation import theodolite.util.PrometheusResponse -import java.time.Instant /** - * A SloChecker can be used to evaluate data from Promehteus. + * A SloChecker can be used to evaluate data from Prometheus. * @constructor Creates an empty SloChecker */ interface SloChecker { - /** * Evaluates [fetchedData] and returns if the experiment was successful. * Returns if the evaluated experiment was successful. @@ -18,5 +16,5 @@ interface SloChecker { * @param fetchedData from Prometheus that will be evaluated. * @return true if experiment was successful. Otherwise false. */ - fun evaluate(start: Instant, end: Instant, fetchedData: PrometheusResponse): Boolean + fun evaluate(fetchedData: List<PrometheusResponse>): Boolean } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt index e179f7fa9492fc4fbe069330046dfd5d83ff8374..e7b511d8c83b5abccece1204aad2a4a9ecfdfd26 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt @@ -24,9 +24,12 @@ abstract class BenchmarkExecutor( val benchmark: Benchmark, val results: Results, val executionDuration: Duration, - configurationOverrides: List<ConfigurationOverride?>, + val configurationOverrides: List<ConfigurationOverride?>, val slo: BenchmarkExecution.Slo, - val executionId: Int + val repetitions: Int, + val executionId: Int, + val loadGenerationDelay: Long, + val afterTeardownDelay: Long ) { var run: AtomicBoolean = AtomicBoolean(true) @@ -41,13 +44,13 @@ abstract class BenchmarkExecutor( * given load, false otherwise. */ abstract fun runExperiment(load: LoadDimension, res: Resource): Boolean - + /** * Wait while the benchmark is running and log the number of minutes executed every 1 minute. * */ fun waitAndLog() { - logger.info { "Execution of a new benchmark started." } + logger.info { "Execution of a new experiment started." } var secondsRunning = 0L diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt index 1ab8e215216cd6f2d3640ca113d438e4ed821042..3afc85f0a8cb67011763498a662b447ce2c07f0f 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt @@ -5,11 +5,9 @@ import mu.KotlinLogging import theodolite.benchmark.Benchmark import theodolite.benchmark.BenchmarkExecution import theodolite.evaluation.AnalysisExecutor -import theodolite.util.ConfigurationOverride -import theodolite.util.LoadDimension -import theodolite.util.Resource -import theodolite.util.Results +import theodolite.util.* import java.time.Duration +import java.time.Instant private val logger = KotlinLogging.logger {} @@ -18,43 +16,58 @@ class BenchmarkExecutorImpl( benchmark: Benchmark, results: Results, executionDuration: Duration, - private val configurationOverrides: List<ConfigurationOverride?>, + configurationOverrides: List<ConfigurationOverride?>, slo: BenchmarkExecution.Slo, - executionId: Int -) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, executionId) { + repetitions: Int, + executionId: Int, + loadGenerationDelay: Long, + afterTeardownDelay: Long +) : BenchmarkExecutor(benchmark, results, executionDuration, configurationOverrides, slo, repetitions, executionId, loadGenerationDelay, afterTeardownDelay) { override fun runExperiment(load: LoadDimension, res: Resource): Boolean { var result = false - val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides) + val executionIntervals: MutableList<Pair<Instant, Instant>> = ArrayList() - try { - benchmarkDeployment.setup() - this.waitAndLog() - } catch (e: Exception) { - logger.error { "Error while setup experiment." } - logger.error { "Error is: $e" } - this.run.set(false) + for (i in 1.rangeTo(repetitions)) { + logger.info { "Run repetition $i/$repetitions" } + if (this.run.get()) { + executionIntervals.add(runSingleExperiment(load,res)) + } else { + break + } } /** * Analyse the experiment, if [run] is true, otherwise the experiment was canceled by the user. */ if (this.run.get()) { - result = - AnalysisExecutor(slo = slo, executionId = executionId).analyze( - load = load, - res = res, - executionDuration = executionDuration - ) + result =AnalysisExecutor(slo = slo, executionId = executionId) + .analyze( + load = load, + res = res, + executionIntervals = executionIntervals) this.results.setResult(Pair(load, res), result) } + return result + } + private fun runSingleExperiment(load: LoadDimension, res: Resource): Pair<Instant, Instant> { + val benchmarkDeployment = benchmark.buildDeployment(load, res, this.configurationOverrides, this.loadGenerationDelay, this.afterTeardownDelay) + val from = Instant.now() + try { + benchmarkDeployment.setup() + this.waitAndLog() + } catch (e: Exception) { + logger.error { "Error while setup experiment." } + logger.error { "Error is: $e" } + this.run.set(false) + } + val to = Instant.now() try { benchmarkDeployment.teardown() } catch (e: Exception) { - logger.warn { "Error while teardown of deplyoment" } - logger.debug { "The Teardowm failed cause of: $e " } + logger.warn { "Error while tearing down the benchmark deployment." } + logger.debug { "Teardown failed, caused by: $e" } } - - return result + return Pair(from,to) } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt index 4518ef7957104819b26eae95cf4e6e9b35c4e995..bf883529967a8b24229fe8256ba0e4edd11b342c 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Main.kt @@ -13,13 +13,17 @@ object Main { @JvmStatic fun main(args: Array<String>) { - val mode = System.getenv("MODE") ?: "yaml-executor" + val mode = System.getenv("MODE") ?: "standalone" logger.info { "Start Theodolite with mode $mode" } - when(mode) { - "yaml-executor" -> TheodoliteYamlExecutor().start() + when (mode) { + "standalone" -> TheodoliteYamlExecutor().start() + "yaml-executor" -> TheodoliteYamlExecutor().start() // TODO remove (#209) "operator" -> TheodoliteOperator().start() - else -> {logger.error { "MODE $mode not found" }; exitProcess(1)} + else -> { + logger.error { "MODE $mode not found" } + exitProcess(1) + } } } -} \ No newline at end of file +} diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt index f5c0251c298dea0801dc601c1d2b790de465459e..0ff8379a0af4b11154214dde021d7c60609631d1 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt @@ -5,6 +5,7 @@ import theodolite.benchmark.BenchmarkExecution import theodolite.benchmark.KubernetesBenchmark import theodolite.util.LoadDimension import theodolite.util.Resource +import java.lang.Exception private val logger = KotlinLogging.logger {} @@ -23,15 +24,23 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b */ override fun run() { // Build Configuration to teardown + try { logger.info { "Received shutdown signal -> Shutting down" } val deployment = benchmark.buildDeployment( load = LoadDimension(0, emptyList()), res = Resource(0, emptyList()), - configurationOverrides = benchmarkExecution.configOverrides + configurationOverrides = benchmarkExecution.configOverrides, + loadGenerationDelay = 0L, + afterTeardownDelay = 5L ) + deployment.teardown() + } catch (e: Exception) { + logger.warn { "Could not delete all specified resources from Kubernetes. " + + "This could be the case, if not all resources are deployed and running." } + + } logger.info { "Teardown everything deployed" } - deployment.teardown() logger.info { "Teardown completed" } } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt index c2b8cc7d2ebc60b7cbb8328dfe1d7830ec5b5aff..c73aaae08489c25a40163d4edb1607247fae010a 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt @@ -1,18 +1,18 @@ package theodolite.execution -import com.google.gson.GsonBuilder +import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution import theodolite.benchmark.KubernetesBenchmark import theodolite.patcher.PatcherDefinitionFactory import theodolite.strategies.StrategyFactory import theodolite.strategies.searchstrategy.CompositeStrategy -import theodolite.util.Config -import theodolite.util.LoadDimension -import theodolite.util.Resource -import theodolite.util.Results -import java.io.PrintWriter +import theodolite.util.* +import java.io.File import java.time.Duration + +private val logger = KotlinLogging.logger {} + /** * The Theodolite executor runs all the experiments defined with the given execution and benchmark configuration. * @@ -62,9 +62,24 @@ class TheodoliteExecutor( executionDuration = executionDuration, configurationOverrides = config.configOverrides, slo = config.slos[0], - executionId = config.executionId + repetitions = config.execution.repetitions, + executionId = config.executionId, + loadGenerationDelay = config.execution.loadGenerationDelay, + afterTeardownDelay = config.execution.afterTeardownDelay ) + if (config.load.loadValues != config.load.loadValues.sorted()) { + config.load.loadValues = config.load.loadValues.sorted() + logger.info { "Load values are not sorted correctly, Theodolite sorts them in ascending order." + + "New order is: ${config.load.loadValues}" } + } + + if (config.resources.resourceValues != config.resources.resourceValues.sorted()) { + config.resources.resourceValues = config.resources.resourceValues.sorted() + logger.info { "Load values are not sorted correctly, Theodolite sorts them in ascending order." + + "New order is: ${config.resources.resourceValues}" } + } + return Config( loads = config.load.loadValues.map { load -> LoadDimension(load, loadDimensionPatcherDefinition) }, resources = config.resources.resourceValues.map { resource -> @@ -97,8 +112,11 @@ class TheodoliteExecutor( * execution and benchmark objects. */ fun run() { - storeAsFile(this.config, "${this.config.executionId}-execution-configuration") - storeAsFile(kubernetesBenchmark, "${this.config.executionId}-benchmark-configuration") + val ioHandler = IOHandler() + val resultsFolder = ioHandler.getResultFolderURL() + this.config.executionId = getAndIncrementExecutionID(resultsFolder+"expID.txt") + ioHandler.writeToJSONFile(this.config, "$resultsFolder${this.config.executionId}-execution-configuration") + ioHandler.writeToJSONFile(kubernetesBenchmark, "$resultsFolder${this.config.executionId}-benchmark-configuration") val config = buildConfig() // execute benchmarks for each load @@ -107,14 +125,17 @@ class TheodoliteExecutor( config.compositeStrategy.findSuitableResource(load, config.resources) } } - storeAsFile(config.compositeStrategy.benchmarkExecutor.results, "${this.config.executionId}-result") + ioHandler.writeToJSONFile(config.compositeStrategy.benchmarkExecutor.results, "$resultsFolder${this.config.executionId}-result") } - private fun <T> storeAsFile(saveObject: T, filename: String) { - val gson = GsonBuilder().enableComplexMapKeySerialization().setPrettyPrinting().create() - - PrintWriter(filename).use { pw -> - pw.println(gson.toJson(saveObject)) - } + private fun getAndIncrementExecutionID(fileURL: String): Int { + val ioHandler = IOHandler() + var executionID = 0 + if (File(fileURL).exists()) { + executionID = ioHandler.readFileAsString(fileURL).toInt() + 1 + } + ioHandler.writeStringToTextFile(fileURL, (executionID).toString()) + return executionID } + } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt index 8b2909f7658f4dffcfd961cad8cd00eb013a160c..b9977029703c8012ada7fb3d7766bfa321a836c3 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt @@ -31,8 +31,8 @@ class TheodoliteYamlExecutor { fun start() { logger.info { "Theodolite started" } - val executionPath = System.getenv("THEODOLITE_EXECUTION") ?: "./config/BenchmarkExecution.yaml" - val benchmarkPath = System.getenv("THEODOLITE_BENCHMARK") ?: "./config/BenchmarkType.yaml" + val executionPath = System.getenv("THEODOLITE_EXECUTION") ?: "./config/example-execution-yaml-resource.yaml" + val benchmarkPath = System.getenv("THEODOLITE_BENCHMARK") ?: "./config/example-benchmark-yaml-resource.yaml" logger.info { "Using $executionPath for BenchmarkExecution" } logger.info { "Using $benchmarkPath for BenchmarkType" } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt new file mode 100644 index 0000000000000000000000000000000000000000..75cbcad051e2055f25d876e66e0fffcdc249c4f5 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt @@ -0,0 +1,55 @@ +package theodolite.execution.operator + +import io.fabric8.kubernetes.api.model.Namespaced +import io.fabric8.kubernetes.client.CustomResource +import io.fabric8.kubernetes.client.CustomResourceDoneable +import io.fabric8.kubernetes.client.CustomResourceList +import io.fabric8.kubernetes.client.KubernetesClient +import io.fabric8.kubernetes.client.dsl.MixedOperation +import io.fabric8.kubernetes.client.dsl.Resource +import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext +import java.lang.Thread.sleep + +abstract class AbstractStateHandler<T,L,D>( + private val context: CustomResourceDefinitionContext, + private val client: KubernetesClient, + private val crd: Class<T>, + private val crdList: Class<L>, + private val donableCRD: Class<D> + ): StateHandler where T : CustomResource, T: Namespaced, L: CustomResourceList<T>, D: CustomResourceDoneable<T> { + + private val crdClient: MixedOperation<T, L, D, Resource<T, D>> = + this.client.customResources(this.context, this.crd, this.crdList, this.donableCRD) + + @Synchronized + override fun setState(resourceName: String, f: (CustomResource) -> CustomResource?) { + this.crdClient + .inNamespace(this.client.namespace) + .list().items + .filter { item -> item.metadata.name == resourceName } + .map { customResource -> f(customResource) } + .forEach { this.crdClient.updateStatus(it as T) } + } + + @Synchronized + override fun getState(resourceName: String, f: (CustomResource) -> String?): String? { + return this.crdClient + .inNamespace(this.client.namespace) + .list().items + .filter { item -> item.metadata.name == resourceName } + .map { customResource -> f(customResource) } + .firstOrNull() + } + + @Synchronized + override fun blockUntilStateIsSet(resourceName: String, desiredStatusString: String, f: (CustomResource) -> String?, maxTries: Int): Boolean { + for (i in 0.rangeTo(maxTries)) { + val currentStatus = getState(resourceName, f) + if(currentStatus == desiredStatusString) { + return true + } + sleep(50) + } + return false + } +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/BenchmarkEventHandler.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/BenchmarkEventHandler.kt deleted file mode 100644 index 69c53a3792d86d0ad1c3e973b1d53ea5defff8d9..0000000000000000000000000000000000000000 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/BenchmarkEventHandler.kt +++ /dev/null @@ -1,66 +0,0 @@ -package theodolite.execution.operator - -import io.fabric8.kubernetes.client.informers.ResourceEventHandler -import mu.KotlinLogging -import theodolite.benchmark.KubernetesBenchmark -private val logger = KotlinLogging.logger {} - -/** - * Handles adding, updating and deleting KubernetesBenchmarks. - * - * @param controller The TheodoliteController that handles the application state - * - * @see TheodoliteController - * @see KubernetesBenchmark - */ -class BenchmarkEventHandler(private val controller: TheodoliteController): ResourceEventHandler<KubernetesBenchmark> { - - /** - * Add a KubernetesBenchmark. - * - * @param benchmark the KubernetesBenchmark to add - * - * @see KubernetesBenchmark - */ - override fun onAdd(benchmark: KubernetesBenchmark) { - benchmark.name = benchmark.metadata.name - logger.info { "Add new benchmark ${benchmark.name}." } - this.controller.benchmarks[benchmark.name] = benchmark - } - - /** - * Update a KubernetesBenchmark. - * - * @param oldBenchmark the KubernetesBenchmark to update - * @param newBenchmark the updated KubernetesBenchmark - * - * @see KubernetesBenchmark - */ - override fun onUpdate(oldBenchmark: KubernetesBenchmark, newBenchmark: KubernetesBenchmark) { - logger.info { "Update benchmark ${newBenchmark.metadata.name}." } - newBenchmark.name = newBenchmark.metadata.name - if (this.controller.isInitialized() && this.controller.executor.getBenchmark().name == oldBenchmark.metadata.name) { - this.controller.isUpdated.set(true) - this.controller.executor.executor.run.compareAndSet(true, false) - } else { - onAdd(newBenchmark) - } - } - - /** - * Delete a KubernetesBenchmark. - * - * @param benchmark the KubernetesBenchmark to delete - * - * @see KubernetesBenchmark - */ - override fun onDelete(benchmark: KubernetesBenchmark, b: Boolean) { - logger.info { "Delete benchmark ${benchmark.metadata.name}." } - this.controller.benchmarks.remove(benchmark.metadata.name) - if ( this.controller.isInitialized() && this.controller.executor.getBenchmark().name == benchmark.metadata.name) { - this.controller.isUpdated.set(true) - this.controller.executor.executor.run.compareAndSet(true, false) - logger.info { "Current benchmark stopped." } - } - } -} diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt new file mode 100644 index 0000000000000000000000000000000000000000..6c8c48f791543b6d8a7716cf26a80bdb449ee7a7 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt @@ -0,0 +1,76 @@ +package theodolite.execution.operator + +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.fabric8.kubernetes.client.dsl.MixedOperation +import io.fabric8.kubernetes.client.dsl.Resource +import mu.KotlinLogging +import org.json.JSONObject +import theodolite.execution.Shutdown +import theodolite.k8s.K8sContextFactory +import theodolite.model.crd.* + +private val logger = KotlinLogging.logger {} + +class ClusterSetup( + private val executionCRDClient: MixedOperation<ExecutionCRD, BenchmarkExecutionList, DoneableExecution, Resource<ExecutionCRD, DoneableExecution>>, + private val benchmarkCRDClient: MixedOperation<BenchmarkCRD, KubernetesBenchmarkList, DoneableBenchmark, Resource<BenchmarkCRD, DoneableBenchmark>>, + private val client: NamespacedKubernetesClient + + ) { + private val serviceMonitorContext = K8sContextFactory().create( + api = "v1", + scope = "Namespaced", + group = "monitoring.coreos.com", + plural = "servicemonitors" + ) + + fun clearClusterState(){ + stopRunningExecution() + clearByLabel() + } + + private fun stopRunningExecution() { + executionCRDClient + .inNamespace(client.namespace) + .list() + .items + .asSequence() + .filter { it.status.executionState == States.RUNNING.value } + .forEach { execution -> + val benchmark = benchmarkCRDClient + .inNamespace(client.namespace) + .list() + .items + .firstOrNull { it.metadata.name == execution.spec.benchmark } + + if (benchmark != null) { + execution.spec.name = execution.metadata.name + benchmark.spec.name = benchmark.metadata.name + Shutdown(execution.spec, benchmark.spec).start() + } else { + logger.error { + "Execution with state ${States.RUNNING.value} was found, but no corresponding benchmark. " + + "Could not initialize cluster." } + } + + + } + } + + private fun clearByLabel() { + this.client.services().withLabel("app.kubernetes.io/created-by=theodolite").delete() + this.client.apps().deployments().withLabel("app.kubernetes.io/created-by=theodolite").delete() + this.client.apps().statefulSets().withLabel("app.kubernetes.io/created-by=theodolite").delete() + this.client.configMaps().withLabel("app.kubernetes.io/created-by=theodolite").delete() + + val serviceMonitors = JSONObject( + this.client.customResource(serviceMonitorContext) + .list(client.namespace, mapOf(Pair("app.kubernetes.io/created-by", "theodolite"))) + ) + .getJSONArray("items") + + (0 until serviceMonitors.length()) + .map { serviceMonitors.getJSONObject(it).getJSONObject("metadata").getString("name") } + .forEach { this.client.customResource(serviceMonitorContext).delete(client.namespace, it) } + } +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt index 971d3428ffde9cf776711bbd68bae68f66597823..a1617b4988d500baab7b02bf5fa993f7a4ae76a3 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionEventHandler.kt @@ -1,9 +1,11 @@ package theodolite.execution.operator +import com.google.gson.Gson +import com.google.gson.GsonBuilder import io.fabric8.kubernetes.client.informers.ResourceEventHandler import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution -import java.lang.NullPointerException +import theodolite.model.crd.* private val logger = KotlinLogging.logger {} @@ -15,56 +17,70 @@ private val logger = KotlinLogging.logger {} * @see TheodoliteController * @see BenchmarkExecution */ -class ExecutionHandler(private val controller: TheodoliteController): ResourceEventHandler<BenchmarkExecution> { +class ExecutionHandler( + private val controller: TheodoliteController, + private val stateHandler: ExecutionStateHandler +) : ResourceEventHandler<ExecutionCRD> { + private val gson: Gson = GsonBuilder().enableComplexMapKeySerialization().create() /** * Add an execution to the end of the queue of the TheodoliteController. * - * @param execution the execution to add + * @param ExecutionCRD the execution to add */ - override fun onAdd(execution: BenchmarkExecution) { - execution.name = execution.metadata.name - logger.info { "Add new execution ${execution.metadata.name} to queue." } - this.controller.executionsQueue.add(execution) + @Synchronized + override fun onAdd(execution: ExecutionCRD) { + logger.info { "Add execution ${execution.metadata.name}" } + execution.spec.name = execution.metadata.name + when (this.stateHandler.getExecutionState(execution.metadata.name)) { + null -> this.stateHandler.setExecutionState(execution.spec.name, States.PENDING) + States.RUNNING -> { + this.stateHandler.setExecutionState(execution.spec.name, States.RESTART) + if(this.controller.isExecutionRunning(execution.spec.name)){ + this.controller.stop(restart=true) + } + } + } } /** - * Update an execution. If this execution is running at the time this function is called, it is stopped and added to - * the beginning of the queue of the TheodoliteController. Otherwise, it is just added to the beginning of the queue. + * Updates an execution. If this execution is running at the time this function is called, it is stopped and + * added to the beginning of the queue of the TheodoliteController. + * Otherwise, it is just added to the beginning of the queue. * - * @param execution the execution to update + * @param oldExecutionCRD the old execution + * @param newExecutionCRD the new execution */ - override fun onUpdate(oldExecution: BenchmarkExecution, newExecution: BenchmarkExecution) { - logger.info { "Add updated execution to queue." } - newExecution.name = newExecution.metadata.name - try { - this.controller.executionsQueue.removeIf { e -> e.name == newExecution.metadata.name } - } catch(e: NullPointerException) { - logger.warn { "No execution found for deletion" } - } - this.controller.executionsQueue.addFirst(newExecution) - if (this.controller.isInitialized() && this.controller.executor.getExecution().name == newExecution.metadata.name) { - this.controller.isUpdated.set(true) - this.controller.executor.executor.run.compareAndSet(true, false) + @Synchronized + override fun onUpdate(oldExecution: ExecutionCRD, newExecution: ExecutionCRD) { + logger.info { "Receive update event for execution ${oldExecution.metadata.name}" } + newExecution.spec.name = newExecution.metadata.name + oldExecution.spec.name = oldExecution.metadata.name + if(gson.toJson(oldExecution.spec) != gson.toJson(newExecution.spec)) { + when(this.stateHandler.getExecutionState(newExecution.metadata.name)) { + States.RUNNING -> { + this.stateHandler.setExecutionState(newExecution.spec.name, States.RESTART) + if (this.controller.isExecutionRunning(newExecution.spec.name)){ + this.controller.stop(restart=true) + } + } + States.RESTART -> {} // should this set to pending? + else -> this.stateHandler.setExecutionState(newExecution.spec.name, States.PENDING) + } + } } - } /** * Delete an execution from the queue of the TheodoliteController. * - * @param execution the execution to delete + * @param ExecutionCRD the execution to delete */ - override fun onDelete(execution: BenchmarkExecution, b: Boolean) { - try { - this.controller.executionsQueue.removeIf { e -> e.name == execution.metadata.name } - logger.info { "Delete execution ${execution.metadata.name} from queue." } - } catch(e: NullPointerException) { - logger.warn { "No execution found for deletion" } - } - if (this.controller.isInitialized() && this.controller.executor.getExecution().name == execution.metadata.name) { - this.controller.isUpdated.set(true) - this.controller.executor.executor.run.compareAndSet(true, false) - logger.info { "Current benchmark stopped." } + @Synchronized + override fun onDelete(execution: ExecutionCRD, b: Boolean) { + logger.info { "Delete execution ${execution.metadata.name}" } + if(execution.status.executionState == States.RUNNING.value + && this.controller.isExecutionRunning(execution.spec.name)) { + this.controller.stop() } } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionStateHandler.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionStateHandler.kt new file mode 100644 index 0000000000000000000000000000000000000000..fe1b95f95c74efe77913ea435dd1ac896805b065 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/ExecutionStateHandler.kt @@ -0,0 +1,82 @@ +package theodolite.execution.operator + +import io.fabric8.kubernetes.client.CustomResource +import io.fabric8.kubernetes.client.KubernetesClient +import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext +import theodolite.model.crd.BenchmarkExecutionList +import theodolite.model.crd.ExecutionCRD +import theodolite.model.crd.States +import java.lang.Thread.sleep +import java.time.Duration +import java.time.Instant +import java.util.concurrent.atomic.AtomicBoolean + +class ExecutionStateHandler(context: CustomResourceDefinitionContext, val client: KubernetesClient): + AbstractStateHandler<ExecutionCRD, BenchmarkExecutionList, DoneableExecution>( + context = context, + client = client, + crd = ExecutionCRD::class.java, + crdList = BenchmarkExecutionList::class.java, + donableCRD = DoneableExecution::class.java) { + + private var runExecutionDurationTimer: AtomicBoolean = AtomicBoolean(false) + + private fun getExecutionLambda() = { cr: CustomResource -> + var execState = "" + if (cr is ExecutionCRD) { execState = cr.status.executionState } + execState + } + + private fun getDurationLambda() = { cr: CustomResource -> + var execState = "" + if (cr is ExecutionCRD) { execState = cr.status.executionState } + execState + } + + fun setExecutionState(resourceName: String, status: States): Boolean { + setState(resourceName) {cr -> if(cr is ExecutionCRD) cr.status.executionState = status.value; cr} + return blockUntilStateIsSet(resourceName, status.value, getExecutionLambda()) + } + + fun getExecutionState(resourceName: String) : States? { + val status = this.getState(resourceName, getExecutionLambda()) + return States.values().firstOrNull { it.value == status } + } + + fun setDurationState(resourceName: String, duration: Duration) { + setState(resourceName) { cr -> if (cr is ExecutionCRD) cr.status.executionDuration = durationToK8sString(duration); cr } + blockUntilStateIsSet(resourceName, durationToK8sString(duration), getDurationLambda()) + } + + fun getDurationState(resourceName: String): String? { + return this.getState(resourceName, getDurationLambda()) + } + + private fun durationToK8sString(duration: Duration): String { + val sec = duration.seconds + return when { + sec <= 120 -> "${sec}s" // max 120s + sec < 60 * 99 -> "${duration.toMinutes()}m" // max 99m + sec < 60 * 60 * 99 -> "${duration.toHours()}h" // max 99h + else -> "${duration.toDays()}d + ${duration.minusDays(duration.toDays()).toHours()}h" + } + } + + fun startDurationStateTimer(resourceName: String) { + this.runExecutionDurationTimer.set(true) + val startTime = Instant.now().toEpochMilli() + Thread { + while (this.runExecutionDurationTimer.get()) { + val duration = Duration.ofMillis(Instant.now().minusMillis(startTime).toEpochMilli()) + setDurationState(resourceName, duration) + sleep(100 * 1) + } + }.start() + } + + @Synchronized + fun stopDurationStateTimer() { + this.runExecutionDurationTimer.set(false) + sleep(100 * 2) + } +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt new file mode 100644 index 0000000000000000000000000000000000000000..9d093e4851e5c43d29a3fea3057ccf01be612e63 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt @@ -0,0 +1,43 @@ +package theodolite.execution.operator + +import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.fabric8.kubernetes.client.extended.leaderelection.LeaderCallbacks +import io.fabric8.kubernetes.client.extended.leaderelection.LeaderElectionConfigBuilder +import io.fabric8.kubernetes.client.extended.leaderelection.resourcelock.LeaseLock +import mu.KotlinLogging +import java.time.Duration +import java.util.* +import kotlin.reflect.KFunction0 + +private val logger = KotlinLogging.logger {} + +class LeaderElector( + val client: NamespacedKubernetesClient, + val name: String + ) { + + fun getLeadership(leader: KFunction0<Unit>) { + val lockIdentity: String = UUID.randomUUID().toString() + DefaultKubernetesClient().use { kc -> + kc.leaderElector() + .withConfig( + LeaderElectionConfigBuilder() + .withName("Theodolite") + .withLeaseDuration(Duration.ofSeconds(15L)) + .withLock(LeaseLock(client.namespace, name, lockIdentity)) + .withRenewDeadline(Duration.ofSeconds(10L)) + .withRetryPeriod(Duration.ofSeconds(2L)) + .withLeaderCallbacks(LeaderCallbacks( + { Thread{leader()}.start() }, + { logger.info { "STOPPED LEADERSHIP" } } + ) { newLeader: String? -> + logger.info { "New leader elected $newLeader" } + }) + .build() + ) + .build().run() + } + } + +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/StateHandler.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/StateHandler.kt new file mode 100644 index 0000000000000000000000000000000000000000..0fbd97e5cca4a9be220eb0b0c89ea0af129a7860 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/StateHandler.kt @@ -0,0 +1,15 @@ +package theodolite.execution.operator + +import io.fabric8.kubernetes.client.CustomResource +private const val MAX_TRIES: Int = 5 + +interface StateHandler { + fun setState(resourceName: String, f: (CustomResource) -> CustomResource?) + fun getState(resourceName: String, f: (CustomResource) -> String?): String? + fun blockUntilStateIsSet( + resourceName: String, + desiredStatusString: String, + f: (CustomResource) -> String?, + maxTries: Int = MAX_TRIES): Boolean + +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt index 532185841a7a8ee000722c1dc513219177f00cae..3053c364e2d6d9bc9797c190f0a87d861089b556 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt @@ -1,123 +1,194 @@ package theodolite.execution.operator import io.fabric8.kubernetes.client.NamespacedKubernetesClient -import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext -import khttp.patch +import io.fabric8.kubernetes.client.dsl.MixedOperation +import io.fabric8.kubernetes.client.dsl.Resource import mu.KotlinLogging import theodolite.benchmark.BenchmarkExecution import theodolite.benchmark.KubernetesBenchmark import theodolite.execution.TheodoliteExecutor +import theodolite.model.crd.* +import theodolite.util.ConfigurationOverride +import theodolite.util.PatcherDefinition import java.lang.Thread.sleep -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ConcurrentLinkedDeque -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicInteger private val logger = KotlinLogging.logger {} /** * The controller implementation for Theodolite. * - * Maintains a Dequeue, based on ConcurrentLinkedDequeue, of executions to be executed for a benchmark. - * - * @param client The NamespacedKubernetesClient - * @param executionContext The CustomResourceDefinitionContext - * * @see NamespacedKubernetesClient * @see CustomResourceDefinitionContext * @see BenchmarkExecution * @see KubernetesBenchmark * @see ConcurrentLinkedDeque */ + class TheodoliteController( - val client: NamespacedKubernetesClient, - val executionContext: CustomResourceDefinitionContext, - val path: String + private val namespace: String, + val path: String, + private val executionCRDClient: MixedOperation<ExecutionCRD, BenchmarkExecutionList, DoneableExecution, Resource<ExecutionCRD, DoneableExecution>>, + private val benchmarkCRDClient: MixedOperation<BenchmarkCRD, KubernetesBenchmarkList, DoneableBenchmark, Resource<BenchmarkCRD, DoneableBenchmark>>, + private val executionStateHandler: ExecutionStateHandler ) { lateinit var executor: TheodoliteExecutor - val executionsQueue: ConcurrentLinkedDeque<BenchmarkExecution> = ConcurrentLinkedDeque() - val benchmarks: ConcurrentHashMap<String, KubernetesBenchmark> = ConcurrentHashMap() - var isUpdated = AtomicBoolean(false) - var executionID = AtomicInteger(0) - /** + * * Runs the TheodoliteController forever. */ fun run() { + sleep(5000) // wait until all states are correctly set while (true) { - try { - reconcile() - logger.info { "Theodolite is waiting for new matching benchmark and execution." } - logger.info { "Currently available executions: " } - executionsQueue.forEach { - logger.info { "${it.name} : waiting for : ${it.benchmark}" } - } - logger.info { "Currently available benchmarks: " } - benchmarks.forEach { - logger.info { it.key } - } - sleep(2000) - } catch (e: InterruptedException) { - logger.error { "Execution interrupted with error: $e." } - } + reconcile() + sleep(2000) } } - /** - * Ensures that the application state corresponds to the defined KubernetesBenchmarks and BenchmarkExecutions. - * - * @see KubernetesBenchmark - * @see BenchmarkExecution - */ - @Synchronized private fun reconcile() { - while (executionsQueue.isNotEmpty()) { - val execution = executionsQueue.peek() - val benchmark = benchmarks[execution.benchmark] - - if (benchmark == null) { - logger.debug { "No benchmark found for execution ${execution.name}." } - sleep(1000) + do { + val execution = getNextExecution() + if (execution != null) { + val benchmark = getBenchmarks() + .firstOrNull { it.name == execution.benchmark } + if (benchmark != null) { + runExecution(execution, benchmark) + } } else { - runExecution(execution, benchmark) + logger.info { "Could not find executable execution." } } - } + } while (execution != null) } /** * Execute a benchmark with a defined KubernetesBenchmark and BenchmarkExecution * - * @see KubernetesBenchmark * @see BenchmarkExecution */ - @Synchronized - fun runExecution(execution: BenchmarkExecution, benchmark: KubernetesBenchmark) { - execution.executionId = executionID.getAndSet(executionID.get() + 1) - isUpdated.set(false) - benchmark.path = path - logger.info { "Start execution ${execution.name} with benchmark ${benchmark.name}." } - executor = TheodoliteExecutor(config = execution, kubernetesBenchmark = benchmark) - executor.run() + private fun runExecution(execution: BenchmarkExecution, benchmark: KubernetesBenchmark) { + setAdditionalLabels(execution.name, + "deployed-for-execution", + benchmark.appResource + benchmark.loadGenResource, + execution) + setAdditionalLabels(benchmark.name, + "deployed-for-benchmark", + benchmark.appResource + benchmark.loadGenResource, + execution) + setAdditionalLabels("theodolite", + "app.kubernetes.io/created-by", + benchmark.appResource + benchmark.loadGenResource, + execution) + + executionStateHandler.setExecutionState(execution.name, States.RUNNING) + executionStateHandler.startDurationStateTimer(execution.name) try { - if (!isUpdated.get()) { - this.executionsQueue.removeIf { e -> e.name == execution.name } - client.customResource(executionContext).delete(client.namespace, execution.metadata.name) + executor = TheodoliteExecutor(execution, benchmark) + executor.run() + when (executionStateHandler.getExecutionState(execution.name)) { + States.RESTART -> runExecution(execution, benchmark) + States.RUNNING -> { + executionStateHandler.setExecutionState(execution.name, States.FINISHED) + logger.info { "Execution of ${execution.name} is finally stopped." } + } } } catch (e: Exception) { - logger.warn { "Deletion skipped." } + logger.error { "Failure while executing execution ${execution.name} with benchmark ${benchmark.name}." } + logger.error { "Problem is: $e" } + executionStateHandler.setExecutionState(execution.name, States.FAILURE) } + executionStateHandler.stopDurationStateTimer() + } - logger.info { "Execution of ${execution.name} is finally stopped." } + @Synchronized + fun stop(restart: Boolean = false) { + if (!::executor.isInitialized) return + if (restart) { + executionStateHandler.setExecutionState(this.executor.getExecution().name, States.RESTART) + } else { + executionStateHandler.setExecutionState(this.executor.getExecution().name, States.INTERRUPTED) + logger.warn { "Execution ${executor.getExecution().name} unexpected interrupted" } + } + this.executor.executor.run.set(false) + } + + /** + * @return all available [BenchmarkCRD]s + */ + private fun getBenchmarks(): List<KubernetesBenchmark> { + return this.benchmarkCRDClient + .inNamespace(namespace) + .list() + .items + .map { it.spec.name = it.metadata.name; it } + .map { it.spec.path = path; it } + .map { it.spec } } /** - * @return true if the TheodoliteExecutor of this controller is initialized. Else returns false. + * Get the [BenchmarkExecution] for the next run. Which [BenchmarkExecution] + * is selected for the next execution depends on three points: * - * @see TheodoliteExecutor + * 1. Only executions are considered for which a matching benchmark is available on the cluster + * 2. The Status of the execution must be [States.PENDING] or [States.RESTART] + * 3. Of the remaining [BenchmarkCRD], those with status [States.RESTART] are preferred, + * then, if there is more than one, the oldest execution is chosen. + * + * @return the next execution or null */ - @Synchronized - fun isInitialized(): Boolean { - return ::executor.isInitialized + private fun getNextExecution(): BenchmarkExecution? { + val availableBenchmarkNames = getBenchmarks() + .map { it.name } + + return executionCRDClient + .inNamespace(namespace) + .list() + .items + .asSequence() + .map { it.spec.name = it.metadata.name; it } + .filter { + it.status.executionState == States.PENDING.value || + it.status.executionState == States.RESTART.value + } + .filter { availableBenchmarkNames.contains(it.spec.benchmark) } + .sortedWith(stateComparator().thenBy { it.metadata.creationTimestamp }) + .map { it.spec } + .firstOrNull() + } + + /** + * Simple comparator which can be used to order a list of [ExecutionCRD] such that executions with + * status [States.RESTART] are before all other executions. + */ + private fun stateComparator() = Comparator<ExecutionCRD> { a, b -> + when { + (a == null && b == null) -> 0 + (a.status.executionState == States.RESTART.value) -> -1 + else -> 1 + } + } + + fun isExecutionRunning(executionName: String): Boolean { + return this.executor.getExecution().name == executionName + } + + private fun setAdditionalLabels( + labelValue: String, + labelName: String, + resources: List<String>, + execution: BenchmarkExecution + ) { + val additionalConfigOverrides = mutableListOf<ConfigurationOverride>() + resources.forEach { + run { + val configurationOverride = ConfigurationOverride() + configurationOverride.patcher = PatcherDefinition() + configurationOverride.patcher.type = "LabelPatcher" + configurationOverride.patcher.properties = mutableMapOf("variableName" to labelName) + configurationOverride.patcher.resource = it + configurationOverride.value = labelValue + additionalConfigOverrides.add(configurationOverride) + } + } + execution.configOverrides.addAll(additionalConfigOverrides) } -} +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt index 5e15a4a80e67f47a42d605d4af39102927139331..0d55b0c1c1c76dba226d34554e0d96a3df77c1c3 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt @@ -1,13 +1,13 @@ package theodolite.execution.operator import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import io.fabric8.kubernetes.client.dsl.MixedOperation +import io.fabric8.kubernetes.client.dsl.Resource import io.fabric8.kubernetes.internal.KubernetesDeserializer import mu.KotlinLogging -import theodolite.benchmark.BenchmarkExecution -import theodolite.benchmark.BenchmarkExecutionList -import theodolite.benchmark.KubernetesBenchmark -import theodolite.benchmark.KubernetesBenchmarkList import theodolite.k8s.K8sContextFactory +import theodolite.model.crd.* private const val DEFAULT_NAMESPACE = "default" @@ -16,7 +16,7 @@ private const val EXECUTION_SINGULAR = "execution" private const val EXECUTION_PLURAL = "executions" private const val BENCHMARK_SINGULAR = "benchmark" private const val BENCHMARK_PLURAL = "benchmarks" -private const val API_VERSION = "v1alpha1" +private const val API_VERSION = "v1" private const val RESYNC_PERIOD = 10 * 60 * 1000.toLong() private const val GROUP = "theodolite.com" private val logger = KotlinLogging.logger {} @@ -28,47 +28,95 @@ private val logger = KotlinLogging.logger {} */ class TheodoliteOperator { private val namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE + val client: NamespacedKubernetesClient = DefaultKubernetesClient().inNamespace(namespace) + + + fun start() { + LeaderElector( + client = client, + name = "theodolite-operator" + ) + .getLeadership(::startOperator) + } /** * Start the operator. */ - fun start() { + private fun startOperator() { logger.info { "Using $namespace as namespace." } - val client = DefaultKubernetesClient().inNamespace(namespace) + client.use { + KubernetesDeserializer.registerCustomKind( + "$GROUP/$API_VERSION", + EXECUTION_SINGULAR, + ExecutionCRD::class.java + ) - KubernetesDeserializer.registerCustomKind( - "$GROUP/$API_VERSION", - EXECUTION_SINGULAR, - BenchmarkExecution::class.java - ) + KubernetesDeserializer.registerCustomKind( + "$GROUP/$API_VERSION", + BENCHMARK_SINGULAR, + BenchmarkCRD::class.java + ) - KubernetesDeserializer.registerCustomKind( - "$GROUP/$API_VERSION", - BENCHMARK_SINGULAR, - KubernetesBenchmark::class.java - ) + val contextFactory = K8sContextFactory() + val executionContext = contextFactory.create(API_VERSION, SCOPE, GROUP, EXECUTION_PLURAL) + val benchmarkContext = contextFactory.create(API_VERSION, SCOPE, GROUP, BENCHMARK_PLURAL) - val contextFactory = K8sContextFactory() - val executionContext = contextFactory.create(API_VERSION, SCOPE, GROUP, EXECUTION_PLURAL) - val benchmarkContext = contextFactory.create(API_VERSION, SCOPE, GROUP, BENCHMARK_PLURAL) + val executionCRDClient: MixedOperation< + ExecutionCRD, + BenchmarkExecutionList, + DoneableExecution, + Resource<ExecutionCRD, DoneableExecution>> + = client.customResources( + executionContext, + ExecutionCRD::class.java, + BenchmarkExecutionList::class.java, + DoneableExecution::class.java) - val appResource = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config" - val controller = TheodoliteController(client = client, executionContext = executionContext, path = appResource) + val benchmarkCRDClient: MixedOperation< + BenchmarkCRD, + KubernetesBenchmarkList, + DoneableBenchmark, + Resource<BenchmarkCRD, DoneableBenchmark>> + = client.customResources( + benchmarkContext, + BenchmarkCRD::class.java, + KubernetesBenchmarkList::class.java, + DoneableBenchmark::class.java) - val informerFactory = client.informers() - val informerExecution = informerFactory.sharedIndexInformerForCustomResource( - executionContext, BenchmarkExecution::class.java, - BenchmarkExecutionList::class.java, RESYNC_PERIOD - ) - val informerBenchmark = informerFactory.sharedIndexInformerForCustomResource( - benchmarkContext, KubernetesBenchmark::class.java, - KubernetesBenchmarkList::class.java, RESYNC_PERIOD - ) + val executionStateHandler = ExecutionStateHandler( + context = executionContext, + client = client) + + val appResource = System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config" + val controller = + TheodoliteController( + namespace = client.namespace, + path = appResource, + benchmarkCRDClient = benchmarkCRDClient, + executionCRDClient = executionCRDClient, + executionStateHandler = executionStateHandler) + + val informerFactory = client.informers() + val informerExecution = informerFactory.sharedIndexInformerForCustomResource( + executionContext, + ExecutionCRD::class.java, + BenchmarkExecutionList::class.java, + RESYNC_PERIOD + ) + + informerExecution.addEventHandler(ExecutionHandler( + controller = controller, + stateHandler = executionStateHandler)) + + ClusterSetup( + executionCRDClient = executionCRDClient, + benchmarkCRDClient = benchmarkCRDClient, + client = client + ).clearClusterState() - informerExecution.addEventHandler(ExecutionHandler(controller)) - informerBenchmark.addEventHandler(BenchmarkEventHandler(controller)) - informerFactory.startAllRegisteredInformers() + informerFactory.startAllRegisteredInformers() + controller.run() - controller.run() + } } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sContextFactory.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sContextFactory.kt index c0e07610171b40c6704602ffa86ec15accb14c19..7eb209bfbab02bb94d34c985aa308173e509d4e4 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sContextFactory.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sContextFactory.kt @@ -21,12 +21,12 @@ class K8sContextFactory { * * @see CustomResourceDefinitionContext */ - fun create(api: String, scope: String, group: String, plural: String ) : CustomResourceDefinitionContext { - return CustomResourceDefinitionContext.Builder() + fun create(api: String, scope: String, group: String, plural: String): CustomResourceDefinitionContext { + return CustomResourceDefinitionContext.Builder() .withVersion(api) .withScope(scope) .withGroup(group) .withPlural(plural) .build() } -} \ No newline at end of file +} diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt index ac2165303f083be066c4398e294e456f1d268dad..c58225bf855c70a5a7057132617418a89b0816a8 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sManager.kt @@ -6,6 +6,9 @@ import io.fabric8.kubernetes.api.model.Service import io.fabric8.kubernetes.api.model.apps.Deployment import io.fabric8.kubernetes.api.model.apps.StatefulSet import io.fabric8.kubernetes.client.NamespacedKubernetesClient +import mu.KotlinLogging + +private val logger = KotlinLogging.logger {} /** * This class is used to deploy or remove different Kubernetes resources. @@ -39,16 +42,32 @@ class K8sManager(private val client: NamespacedKubernetesClient) { */ fun remove(resource: KubernetesResource) { when (resource) { - is Deployment -> + is Deployment -> { + val label = resource.spec.selector.matchLabels["app"]!! this.client.apps().deployments().delete(resource) + blockUntilPodsDeleted(label) + logger.info { "Deployment '${resource.metadata.name}' deleted." } + } is Service -> this.client.services().delete(resource) is ConfigMap -> this.client.configMaps().delete(resource) - is StatefulSet -> + is StatefulSet -> { + val label = resource.spec.selector.matchLabels["app"]!! this.client.apps().statefulSets().delete(resource) + blockUntilPodsDeleted(label) + logger.info { "StatefulSet '$resource.metadata.name' deleted." } + } is ServiceMonitorWrapper -> resource.delete(client) else -> throw IllegalArgumentException("Unknown Kubernetes resource.") } } + + private fun blockUntilPodsDeleted(podLabel: String) { + while (!this.client.pods().withLabel(podLabel).list().items.isNullOrEmpty()) { + logger.info { "Wait for pods with label '$podLabel' to be deleted." } + Thread.sleep(1000) + } + } + } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt index 324b02b74b2c53eb1292667f037f3fdbcc114b73..fd5ec3b2e744b5e096187626a2fd756f71f87e67 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt @@ -32,7 +32,14 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) { * @return CustomResource from fabric8 */ private fun loadServiceMonitor(path: String): ServiceMonitorWrapper { - return loadGenericResource(path) { x: String -> ServiceMonitorWrapper(YamlParser().parse(path, HashMap<String, String>()::class.java)!!) } + return loadGenericResource(path) { x: String -> + ServiceMonitorWrapper( + YamlParser().parse( + path, + HashMap<String, String>()::class.java + )!! + ) + } } /** diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/ServiceMonitorWrapper.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/ServiceMonitorWrapper.kt index 4950cee225e103ff095def91de64471ec1894a79..56452d74968db0fd4c939f44f3ed8a7abbe7b928 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/ServiceMonitorWrapper.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/ServiceMonitorWrapper.kt @@ -53,4 +53,9 @@ class ServiceMonitorWrapper(private val serviceMonitor: Map<String, String>) : C val smAsMap = this.serviceMonitor["metadata"]!! as Map<String, String> return smAsMap["name"]!! } + + fun getLabels(): Map<String, String>{ + val smAsMap = this.serviceMonitor["metadata"]!! as Map<String, String> + return smAsMap["labels"]!! as Map<String, String> + } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt index 321fa9a945fade012a81ea4dbacb2c403e783411..3bbae82d77dc5b01a5827c7ee713bf2566be1bab 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/k8s/TopicManager.kt @@ -4,6 +4,7 @@ import mu.KotlinLogging import org.apache.kafka.clients.admin.AdminClient import org.apache.kafka.clients.admin.CreateTopicsResult import org.apache.kafka.clients.admin.NewTopic +import org.apache.kafka.common.errors.TopicExistsException import java.lang.Thread.sleep private val logger = KotlinLogging.logger {} @@ -11,13 +12,14 @@ private const val RETRY_TIME = 2000L /** * Manages the topics related tasks - * @param kafkaConfig Kafka Configuration as HashMap + * @param kafkaConfig Kafka configuration as a Map * @constructor Creates a KafkaAdminClient */ -class TopicManager(private val kafkaConfig: HashMap<String, Any>) { +class TopicManager(private val kafkaConfig: Map<String, Any>) { + /** - * Creates topics. - * @param newTopics List of all Topic that should be created + * Create topics. + * @param newTopics Collection of all topic that should be created */ fun createTopics(newTopics: Collection<NewTopic>) { val kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig) @@ -27,11 +29,13 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { var retryCreation = false try { result = kafkaAdmin.createTopics(newTopics) - result.all().get()// wait for the future object - - } catch (e: Exception) { - logger.warn { "Error during topic creation." } - logger.warn { "Will retry the topic creation after 2 seconds" } + result.all().get() // wait for the future to be completed + } catch (e: Exception) { // TopicExistsException + logger.warn(e) { "Error during topic creation." } + logger.debug { e } // TODO remove due to attached exception to warn log? + logger.info { "Remove existing topics." } + delete(newTopics.map { topic -> topic.name() }, kafkaAdmin) + logger.info { "Will retry the topic creation in ${RETRY_TIME/1000} seconds." } sleep(RETRY_TIME) retryCreation = true } @@ -39,7 +43,9 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { logger.info { "Topics creation finished with result: ${ - result.values().map { it -> it.key + ": " + it.value.isDone } + result + .values() + .map { it.key + ": " + it.value.isDone } .joinToString(separator = ",") } " } @@ -47,39 +53,61 @@ class TopicManager(private val kafkaConfig: HashMap<String, Any>) { } /** - * Removes topics. - * @param topics List of names with the topics to remove. + * Remove topics. + * @param topics Collection of names for the topics to remove. */ fun removeTopics(topics: List<String>) { val kafkaAdmin: AdminClient = AdminClient.create(this.kafkaConfig) + val currentTopics = kafkaAdmin.listTopics().names().get() + delete(currentTopics.filter { matchRegex(it, topics) }, kafkaAdmin) + kafkaAdmin.close() + } + + /** + * This function checks whether one string in `topics` can be used as prefix of a regular expression + * to create the string `existingTopic`. + * + * @param existingTopic string for which should be checked if it could be created. + * @param topics list of string which are used as possible prefixes to create `existingTopic`. + * @return true, `existingTopics` matches a created regex, else false. + */ + private fun matchRegex(existingTopic: String, topics: List<String>): Boolean { + for (t in topics) { + val regex = t.toRegex() + if (regex.matches(existingTopic)) { + return true + } + } + return false + } + + private fun delete(topics: List<String>, kafkaAdmin: AdminClient) { var deleted = false - + while (!deleted) { try { val result = kafkaAdmin.deleteTopics(topics) - result.all().get() // wait for the future object + result.all().get() // wait for the future to be completed logger.info { "Topics deletion finished with result: ${ - result.values().map { it -> it.key + ": " + it.value.isDone } + result.values().map { it.key + ": " + it.value.isDone } .joinToString(separator = ",") }" } } catch (e: Exception) { - logger.error { "Error while removing topics: $e" } - logger.debug { "Existing topics are: ${kafkaAdmin.listTopics()}." } + logger.error(e) { "Error while removing topics: $e" } + logger.info { "Existing topics are: ${kafkaAdmin.listTopics().names().get()}." } } - val toDelete = topics.filter { topic -> - kafkaAdmin.listTopics().names().get().contains(topic) - } + val toDelete = topics.filter { kafkaAdmin.listTopics().names().get().contains(it) } if (toDelete.isNullOrEmpty()) { deleted = true } else { - logger.info { "Deletion of kafka topics failed retrying in 2 seconds" } + logger.info { "Deletion of Kafka topics failed, will retry in ${RETRY_TIME/1000} seconds." } sleep(RETRY_TIME) } } - kafkaAdmin.close() } + } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/BenchmarkCRD.kt b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/BenchmarkCRD.kt new file mode 100644 index 0000000000000000000000000000000000000000..326aa10a21bebd913eaafcb8315188288ae97ff1 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/BenchmarkCRD.kt @@ -0,0 +1,11 @@ +package theodolite.model.crd + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.Namespaced +import io.fabric8.kubernetes.client.CustomResource +import theodolite.benchmark.KubernetesBenchmark + +@JsonDeserialize +class BenchmarkCRD( + var spec: KubernetesBenchmark = KubernetesBenchmark() +) : CustomResource(), Namespaced \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/BenchmarkExecutionList.kt b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/BenchmarkExecutionList.kt new file mode 100644 index 0000000000000000000000000000000000000000..2b2dcc07f9c37f1712109e3d092f2db0c139e1c8 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/BenchmarkExecutionList.kt @@ -0,0 +1,5 @@ +package theodolite.model.crd + +import io.fabric8.kubernetes.client.CustomResourceList + +class BenchmarkExecutionList : CustomResourceList<ExecutionCRD>() diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/DoneableBenchmark.kt b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/DoneableBenchmark.kt new file mode 100644 index 0000000000000000000000000000000000000000..e00e8268b2ec8eba17b3706feb3940eded1b2b0c --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/DoneableBenchmark.kt @@ -0,0 +1,7 @@ +package theodolite.model.crd + +import io.fabric8.kubernetes.api.builder.Function +import io.fabric8.kubernetes.client.CustomResourceDoneable + +class DoneableBenchmark(resource: BenchmarkCRD, function: Function<BenchmarkCRD, BenchmarkCRD>) : + CustomResourceDoneable<BenchmarkCRD>(resource, function) \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/DoneableExecution.kt b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/DoneableExecution.kt new file mode 100644 index 0000000000000000000000000000000000000000..be07725b405c29a0d9000b6e6ec455536ad111fb --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/DoneableExecution.kt @@ -0,0 +1,8 @@ +package theodolite.execution.operator + +import io.fabric8.kubernetes.client.CustomResourceDoneable +import io.fabric8.kubernetes.api.builder.Function +import theodolite.model.crd.ExecutionCRD + +class DoneableExecution(resource: ExecutionCRD, function: Function<ExecutionCRD, ExecutionCRD>) : + CustomResourceDoneable<ExecutionCRD>(resource, function) \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/ExecutionCRD.kt b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/ExecutionCRD.kt new file mode 100644 index 0000000000000000000000000000000000000000..79a387cee250d3abf0fdb576a5f0f33424596792 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/ExecutionCRD.kt @@ -0,0 +1,13 @@ +package theodolite.model.crd + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.api.model.Namespaced +import io.fabric8.kubernetes.client.CustomResource +import theodolite.benchmark.BenchmarkExecution + +@JsonDeserialize +class ExecutionCRD( + var spec: BenchmarkExecution = BenchmarkExecution(), + var status: ExecutionStatus = ExecutionStatus() + ) : CustomResource(), Namespaced \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/ExecutionStatus.kt b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/ExecutionStatus.kt new file mode 100644 index 0000000000000000000000000000000000000000..43e9035b3120eb22304576f2006092eec376b6d2 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/ExecutionStatus.kt @@ -0,0 +1,13 @@ +package theodolite.model.crd + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.api.model.Namespaced +import io.fabric8.kubernetes.client.CustomResource + +@JsonDeserialize +class ExecutionStatus(): KubernetesResource, CustomResource(), Namespaced { + var executionState: String = "" + var executionDuration: String = "-" + +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/KubernetesBenchmarkList.kt b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/KubernetesBenchmarkList.kt new file mode 100644 index 0000000000000000000000000000000000000000..8ad0a493d948bf5f78741052100766dcf6e316ec --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/KubernetesBenchmarkList.kt @@ -0,0 +1,5 @@ +package theodolite.model.crd + +import io.fabric8.kubernetes.client.CustomResourceList + +class KubernetesBenchmarkList : CustomResourceList<BenchmarkCRD>() diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/States.kt b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/States.kt new file mode 100644 index 0000000000000000000000000000000000000000..79af297915b6703b209acb0c13913482e54db2be --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/model/crd/States.kt @@ -0,0 +1,11 @@ +package theodolite.model.crd + +enum class States(val value: String) { + RUNNING("RUNNING"), + PENDING("PENDING"), + FAILURE("FAILURE"), + FINISHED("FINISHED"), + RESTART("RESTART"), + INTERRUPTED("INTERRUPTED"), + NO_STATE("NoState") +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/AbstractPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/AbstractPatcher.kt index a1a4501c919748389089b9d81e3cf927b0ea2e2a..df80e9cbd2503685a7dbed35db5319920dfc42cb 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/AbstractPatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/AbstractPatcher.kt @@ -12,7 +12,7 @@ import io.fabric8.kubernetes.api.model.KubernetesResource * @param variableName *(optional)* The variable name to be patched * * - * **For example** to patch the load dimension of a workload generator, the Patcher should be created as follow: + * **For example** to patch the load dimension of a load generator, the patcher should be created as follow: * * k8sResource: `uc-1-workload-generator.yaml` * container: `workload` @@ -20,7 +20,5 @@ import io.fabric8.kubernetes.api.model.KubernetesResource * */ abstract class AbstractPatcher( - k8sResource: KubernetesResource, - container: String? = null, - variableName: String? = null + k8sResource: KubernetesResource ) : Patcher diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/EnvVarPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/EnvVarPatcher.kt index 16bd9aa34127b79c97e8f9d195d4757145a3fa93..416aec74a3af9b74594f5e6cd018682bf91cbf63 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/EnvVarPatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/EnvVarPatcher.kt @@ -2,7 +2,6 @@ package theodolite.patcher import io.fabric8.kubernetes.api.model.Container import io.fabric8.kubernetes.api.model.EnvVar -import io.fabric8.kubernetes.api.model.EnvVarSource import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.api.model.apps.Deployment @@ -17,7 +16,7 @@ class EnvVarPatcher( private val k8sResource: KubernetesResource, private val container: String, private val variableName: String -) : AbstractPatcher(k8sResource, container, variableName) { +) : AbstractPatcher(k8sResource) { override fun <String> patch(value: String) { if (k8sResource is Deployment) { @@ -39,7 +38,9 @@ class EnvVarPatcher( val x = container.env.filter { envVar -> envVar.name == k } if (x.isEmpty()) { - val newVar = EnvVar(k, v, EnvVarSource()) + val newVar = EnvVar() + newVar.name = k + newVar.value = v container.env.add(newVar) } else { x.forEach { diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ImagePatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ImagePatcher.kt index e5e5f6cb67641c71ad0fd31375752cbb03fa62db..8f6753372076c119324dc962112928253633b6b0 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ImagePatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ImagePatcher.kt @@ -11,8 +11,8 @@ import io.fabric8.kubernetes.api.model.apps.StatefulSet * @param container Container to be patched. */ class ImagePatcher(private val k8sResource: KubernetesResource, private val container: String) : - AbstractPatcher(k8sResource, container) { - + AbstractPatcher(k8sResource) { + override fun <String> patch(imagePath: String) { if (k8sResource is Deployment) { k8sResource.spec.template.spec.containers.filter { it.name == container }.forEach { diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/LabelPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/LabelPatcher.kt new file mode 100644 index 0000000000000000000000000000000000000000..5ee5807cd8378c9f2bbd62435203208d61131f15 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/LabelPatcher.kt @@ -0,0 +1,49 @@ +package theodolite.patcher + +import io.fabric8.kubernetes.api.model.ConfigMap +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.api.model.Service +import io.fabric8.kubernetes.api.model.apps.Deployment +import io.fabric8.kubernetes.api.model.apps.StatefulSet +import io.fabric8.kubernetes.client.CustomResource + +class LabelPatcher(private val k8sResource: KubernetesResource, val variableName: String) : + AbstractPatcher(k8sResource) { + + override fun <String> patch(labelValue: String) { + if(labelValue is kotlin.String){ + when(k8sResource){ + is Deployment -> { + if (k8sResource.metadata.labels == null){ + k8sResource.metadata.labels = mutableMapOf() + } + k8sResource.metadata.labels[this.variableName] = labelValue + } + is StatefulSet -> { + if (k8sResource.metadata.labels == null){ + k8sResource.metadata.labels = mutableMapOf() + } + k8sResource.metadata.labels[this.variableName] = labelValue + } + is Service -> { + if (k8sResource.metadata.labels == null){ + k8sResource.metadata.labels = mutableMapOf() + } + k8sResource.metadata.labels[this.variableName] = labelValue + } + is ConfigMap -> { + if (k8sResource.metadata.labels == null){ + k8sResource.metadata.labels = mutableMapOf() + } + k8sResource.metadata.labels[this.variableName] = labelValue + } + is CustomResource -> { + if (k8sResource.metadata.labels == null){ + k8sResource.metadata.labels = mutableMapOf() + } + k8sResource.metadata.labels[this.variableName] = labelValue + } + } + } + } +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NodeSelectorPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NodeSelectorPatcher.kt index 0a668a908e66577f96ea1268b85a38ad73bb16a7..0e8cd553a6c6a9ed6fa2c8cc1b84e4cfebe79d73 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NodeSelectorPatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NodeSelectorPatcher.kt @@ -10,7 +10,7 @@ import io.fabric8.kubernetes.api.model.apps.Deployment * @param variableName The `label-key` of the node for which the `label-value` is to be patched. */ class NodeSelectorPatcher(private val k8sResource: KubernetesResource, private val variableName: String) : - AbstractPatcher(k8sResource, variableName) { + AbstractPatcher(k8sResource) { override fun <String> patch(value: String) { if (k8sResource is Deployment) { k8sResource.spec.template.spec.nodeSelector = mapOf(variableName to value as kotlin.String) diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt index 7cf56f8452949e387a186aa8f8c962e1ee1aad15..65489a96974ad566fe7cbd88cf6ff7fb49135e1d 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumNestedGroupsLoadGeneratorReplicaPatcher.kt @@ -4,15 +4,17 @@ import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.api.model.apps.Deployment import kotlin.math.pow -private const val NUM_SENSORS = 4.0 -private const val LOAD_GEN_MAX_RECORDS = 150000 - -class NumNestedGroupsLoadGeneratorReplicaPatcher(private val k8sResource: KubernetesResource) : AbstractPatcher(k8sResource) { +class NumNestedGroupsLoadGeneratorReplicaPatcher( + private val k8sResource: KubernetesResource, + private val numSensors: String, + private val loadGenMaxRecords: String + ) : + AbstractPatcher(k8sResource) { override fun <String> patch(value: String) { if (k8sResource is Deployment) { if (value is kotlin.String) { - val approxNumSensors = NUM_SENSORS.pow(Integer.parseInt(value).toDouble()) - val loadGenInstances = (approxNumSensors + LOAD_GEN_MAX_RECORDS -1) / LOAD_GEN_MAX_RECORDS + val approxNumSensors = numSensors.toDouble().pow(Integer.parseInt(value).toDouble()) + val loadGenInstances = (approxNumSensors + loadGenMaxRecords.toDouble() - 1) / loadGenMaxRecords.toDouble() this.k8sResource.spec.replicas = loadGenInstances.toInt() } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt index 6f2ebcb8b1eb37801c7f6bb2f28c251a07ae44e8..f6a06324e36d7942d3944a492fee263f428376c1 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/NumSensorsLoadGeneratorReplicaPatcher.kt @@ -3,13 +3,16 @@ package theodolite.patcher import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.api.model.apps.Deployment -private const val LOAD_GEN_MAX_RECORDS = 150000 -class NumSensorsLoadGeneratorReplicaPatcher(private val k8sResource: KubernetesResource) : AbstractPatcher(k8sResource) { +class NumSensorsLoadGeneratorReplicaPatcher( + private val k8sResource: KubernetesResource, + private val loadGenMaxRecords: String +) : + AbstractPatcher(k8sResource) { override fun <String> patch(value: String) { if (k8sResource is Deployment) { if (value is kotlin.String) { - val loadGenInstances = (Integer.parseInt(value) + LOAD_GEN_MAX_RECORDS - 1) / LOAD_GEN_MAX_RECORDS + val loadGenInstances = (Integer.parseInt(value) + loadGenMaxRecords.toInt() - 1) / loadGenMaxRecords.toInt() this.k8sResource.spec.replicas = loadGenInstances } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherDefinitionFactory.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherDefinitionFactory.kt index bcb568f716449cb1981112ab23a81411a0f7c54d..d5a6f3821d2688651475625506a78efc6061ab82 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherDefinitionFactory.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherDefinitionFactory.kt @@ -21,8 +21,7 @@ class PatcherDefinitionFactory { * value of the requiredType. */ fun createPatcherDefinition(requiredType: String, patcherTypes: List<TypeName>): List<PatcherDefinition> { - return patcherTypes - .filter { type -> type.typeName == requiredType } - .flatMap { type -> type.patchers } + return patcherTypes.firstOrNull() { type -> type.typeName == requiredType } + ?.patchers ?: throw IllegalArgumentException("typeName $requiredType not found.") } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherFactory.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherFactory.kt index 2ee1f6c7b46322cb0f8de03c37aabe64ccf0ba5a..45e50113c964d671962fadc718994a29b2da81f4 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherFactory.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/PatcherFactory.kt @@ -1,6 +1,8 @@ package theodolite.patcher import io.fabric8.kubernetes.api.model.KubernetesResource +import theodolite.util.DeploymentFailedException +import theodolite.util.InvalidPatcherConfigurationException import theodolite.util.PatcherDefinition /** @@ -27,25 +29,56 @@ class PatcherFactory { k8sResources: List<Pair<String, KubernetesResource>> ): Patcher { val resource = - k8sResources.filter { it.first == patcherDefinition.resource }.map { resource -> resource.second }[0] - return when (patcherDefinition.type) { - "ReplicaPatcher" -> ReplicaPatcher(resource) - "NumNestedGroupsLoadGeneratorReplicaPatcher" -> NumNestedGroupsLoadGeneratorReplicaPatcher(resource) - "NumSensorsLoadGeneratorReplicaPatcher" -> NumSensorsLoadGeneratorReplicaPatcher(resource) - "EnvVarPatcher" -> EnvVarPatcher(resource, patcherDefinition.container, patcherDefinition.variableName) - "NodeSelectorPatcher" -> NodeSelectorPatcher(resource, patcherDefinition.variableName) - "ResourceLimitPatcher" -> ResourceLimitPatcher( - resource, - patcherDefinition.container, - patcherDefinition.variableName - ) - "ResourceRequestPatcher" -> ResourceRequestPatcher( - resource, - patcherDefinition.container, - patcherDefinition.variableName - ) - "SchedulerNamePatcher" -> SchedulerNamePatcher(resource) - else -> throw IllegalArgumentException("Patcher type ${patcherDefinition.type} not found") + k8sResources.filter { it.first == patcherDefinition.resource } + .map { resource -> resource.second } + .firstOrNull() + ?: throw DeploymentFailedException("Could not find resource ${patcherDefinition.resource}") + + return try { + when (patcherDefinition.type) { + "ReplicaPatcher" -> ReplicaPatcher( + k8sResource = resource + ) + "NumNestedGroupsLoadGeneratorReplicaPatcher" -> NumNestedGroupsLoadGeneratorReplicaPatcher( + k8sResource = resource, + loadGenMaxRecords = patcherDefinition.properties["loadGenMaxRecords"] !!, + numSensors = patcherDefinition.properties["numSensors"] !! + ) + "NumSensorsLoadGeneratorReplicaPatcher" -> NumSensorsLoadGeneratorReplicaPatcher( + k8sResource = resource, + loadGenMaxRecords = patcherDefinition.properties["loadGenMaxRecords"] !! + ) + "EnvVarPatcher" -> EnvVarPatcher( + k8sResource = resource, + container = patcherDefinition.properties["container"] !!, + variableName = patcherDefinition.properties["variableName"] !! + ) + "NodeSelectorPatcher" -> NodeSelectorPatcher( + k8sResource = resource, + variableName = patcherDefinition.properties["variableName"] !! + ) + "ResourceLimitPatcher" -> ResourceLimitPatcher( + k8sResource = resource, + container = patcherDefinition.properties["container"] !!, + limitedResource = patcherDefinition.properties["limitedResource"] !! + ) + "ResourceRequestPatcher" -> ResourceRequestPatcher( + k8sResource = resource, + container = patcherDefinition.properties["container"] !!, + requestedResource = patcherDefinition.properties["requestedResource"] !! + ) + "SchedulerNamePatcher" -> SchedulerNamePatcher( + k8sResource = resource + ) + "LabelPatcher" -> LabelPatcher( + k8sResource = resource, + variableName = patcherDefinition.properties["variableName"] !! + ) + else -> throw InvalidPatcherConfigurationException("Patcher type ${patcherDefinition.type} not found.") + } + } catch (e: Exception) { + throw InvalidPatcherConfigurationException("Could not create patcher with type ${patcherDefinition.type}" + + " Probably a required patcher argument was not specified." ) } } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ResourceLimitPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ResourceLimitPatcher.kt index eab82effbc084e91ba57c1bea7103b2a3239c922..1a6fa35a944d00634ec0607b0bff34f4cb9d9b9c 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ResourceLimitPatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ResourceLimitPatcher.kt @@ -6,7 +6,6 @@ import io.fabric8.kubernetes.api.model.Quantity import io.fabric8.kubernetes.api.model.ResourceRequirements import io.fabric8.kubernetes.api.model.apps.Deployment import io.fabric8.kubernetes.api.model.apps.StatefulSet -import java.lang.IllegalArgumentException /** * The Resource limit [Patcher] set resource limits for deployments and statefulSets. @@ -19,7 +18,7 @@ class ResourceLimitPatcher( private val k8sResource: KubernetesResource, private val container: String, private val limitedResource: String -) : AbstractPatcher(k8sResource, container, limitedResource) { +) : AbstractPatcher(k8sResource) { override fun <String> patch(value: String) { when (k8sResource) { diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ResourceRequestPatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ResourceRequestPatcher.kt index f4ef38edebab67022066394e149716ab9ffbce00..9bf8c3c72f656d326ca3070cd5843778e5cdff42 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ResourceRequestPatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/ResourceRequestPatcher.kt @@ -6,7 +6,6 @@ import io.fabric8.kubernetes.api.model.Quantity import io.fabric8.kubernetes.api.model.ResourceRequirements import io.fabric8.kubernetes.api.model.apps.Deployment import io.fabric8.kubernetes.api.model.apps.StatefulSet -import java.lang.IllegalArgumentException /** * The Resource request [Patcher] set resource limits for deployments and statefulSets. @@ -19,7 +18,7 @@ class ResourceRequestPatcher( private val k8sResource: KubernetesResource, private val container: String, private val requestedResource: String -) : AbstractPatcher(k8sResource, container, requestedResource) { +) : AbstractPatcher(k8sResource) { override fun <String> patch(value: String) { when (k8sResource) { diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/SchedulerNamePatcher.kt b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/SchedulerNamePatcher.kt index 589bceff78158a422d923169bd35a1e11e2f4caa..348f0c50090a34c91221d3e099c3532375a578da 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/patcher/SchedulerNamePatcher.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/patcher/SchedulerNamePatcher.kt @@ -4,14 +4,14 @@ import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.api.model.apps.Deployment /** - * The Scheduler name [Patcher] make it possible to set the scheduler which should be used to deploy the given deployment. - * + * The Scheduler name [Patcher] make it possible to set the scheduler which should + * be used to deploy the given deployment. * @param k8sResource Kubernetes resource to be patched. */ -class SchedulerNamePatcher(private val k8sResource: KubernetesResource): Patcher { +class SchedulerNamePatcher(private val k8sResource: KubernetesResource) : Patcher { override fun <String> patch(value: String) { if (k8sResource is Deployment) { k8sResource.spec.template.spec.schedulerName = value as kotlin.String } } -} \ No newline at end of file +} diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/StrategyFactory.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/StrategyFactory.kt index 9bef5587ac9c26d2323af41c5119ac36b95cf807..829370e8ce1c181c1a4cb9fdd8ccf0ecefd48d3d 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/StrategyFactory.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/StrategyFactory.kt @@ -4,12 +4,13 @@ import theodolite.execution.BenchmarkExecutor import theodolite.strategies.restriction.LowerBoundRestriction import theodolite.strategies.restriction.RestrictionStrategy import theodolite.strategies.searchstrategy.BinarySearch +import theodolite.strategies.searchstrategy.FullSearch import theodolite.strategies.searchstrategy.LinearSearch import theodolite.strategies.searchstrategy.SearchStrategy import theodolite.util.Results /** - * Factory for creating [SearchStrategy] and [RestrictionStrategy] Strategies. + * Factory for creating [SearchStrategy] and [RestrictionStrategy] strategies. */ class StrategyFactory { @@ -24,6 +25,7 @@ class StrategyFactory { */ fun createSearchStrategy(executor: BenchmarkExecutor, searchStrategyString: String): SearchStrategy { return when (searchStrategyString) { + "FullSearch" -> FullSearch(executor) "LinearSearch" -> LinearSearch(executor) "BinarySearch" -> BinarySearch(executor) else -> throw IllegalArgumentException("Search Strategy $searchStrategyString not found") diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/restriction/LowerBoundRestriction.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/restriction/LowerBoundRestriction.kt index 2911b6ac949a9d523e464c0ea2942063e996d767..13bfedfe055f2bd428137f89b2986f3967ec797c 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/restriction/LowerBoundRestriction.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/restriction/LowerBoundRestriction.kt @@ -5,12 +5,13 @@ import theodolite.util.Resource import theodolite.util.Results /** - * The Lower Bound Restriction sets the lower bound of the resources to be examined to the value + * The [LowerBoundRestriction] sets the lower bound of the resources to be examined to the value * needed to successfully execute the next smaller load. * * @param results [Result] object used as a basis to restrict the resources. */ class LowerBoundRestriction(results: Results) : RestrictionStrategy(results) { + override fun apply(load: LoadDimension, resources: List<Resource>): List<Resource> { val maxLoad: LoadDimension? = this.results.getMaxBenchmarkedLoad(load) var lowerBound: Resource? = this.results.getMinRequiredInstances(maxLoad) @@ -19,4 +20,5 @@ class LowerBoundRestriction(results: Results) : RestrictionStrategy(results) { } return resources.filter { x -> x.get() >= lowerBound.get() } } + } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/BinarySearch.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/BinarySearch.kt index 027444fe36a47878af998abdf18dc7a7562d7afd..28e8194c699cd074026c8cb7e6f3ce4ec347023b 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/BinarySearch.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/BinarySearch.kt @@ -1,9 +1,12 @@ package theodolite.strategies.searchstrategy +import mu.KotlinLogging import theodolite.execution.BenchmarkExecutor import theodolite.util.LoadDimension import theodolite.util.Resource +private val logger = KotlinLogging.logger {} + /** * Binary-search-like implementation for determining the smallest suitable number of instances. * @@ -32,6 +35,8 @@ class BinarySearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchm } // special case: length == 1 or 2 if (lower == upper) { + val res = resources[lower] + logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" } if (this.benchmarkExecutor.runExperiment(load, resources[lower])) return lower else { if (lower + 1 == resources.size) return -1 @@ -41,6 +46,8 @@ class BinarySearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchm // apply binary search for a list with // length > 2 and adjust upper and lower depending on the result for `resources[mid]` val mid = (upper + lower) / 2 + val res = resources[mid] + logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" } if (this.benchmarkExecutor.runExperiment(load, resources[mid])) { if (mid == lower) { return lower diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/CompositeStrategy.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/CompositeStrategy.kt index 6ae06d70c9effe0a0a4bbd9abffa665fb08636c9..41cc5c325163ade54469398e815fdb8d95c6e6cd 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/CompositeStrategy.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/CompositeStrategy.kt @@ -10,7 +10,7 @@ import theodolite.util.Resource * Composite strategy that combines a SearchStrategy and a set of RestrictionStrategy. * * @param searchStrategy the [SearchStrategy] that is executed as part of this [CompositeStrategy]. - * @param restrictionStrategies the set of [RestrictionStrategy] that are connected conjuntively to restrict the [Resource] + * @param restrictionStrategies the set of [RestrictionStrategy] that are connected conjunctive to restrict the [Resource] * @param benchmarkExecutor Benchmark executor which runs the individual benchmarks. */ @RegisterForReflection diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/FullSearch.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/FullSearch.kt new file mode 100644 index 0000000000000000000000000000000000000000..cb0dd2d8ab528e42e8290f59f26c8b9b32f384c7 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/FullSearch.kt @@ -0,0 +1,31 @@ +package theodolite.strategies.searchstrategy + +import mu.KotlinLogging +import theodolite.execution.BenchmarkExecutor +import theodolite.util.LoadDimension +import theodolite.util.Resource + +private val logger = KotlinLogging.logger {} + +/** + * [SearchStrategy] that executes experiment for provides resources in a linear-search-like fashion, but **without + * stopping** once a suitable resource amount is found. + * + * @see LinearSearch for a SearchStrategy that stops once a suitable resource amount is found. + * + * @param benchmarkExecutor Benchmark executor which runs the individual benchmarks. + */ +class FullSearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchmarkExecutor) { + + override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? { + var minimalSuitableResources: Resource? = null + for (res in resources) { + logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" } + val result = this.benchmarkExecutor.runExperiment(load, res) + if (result && minimalSuitableResources != null) { + minimalSuitableResources = res + } + } + return minimalSuitableResources + } +} diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/LinearSearch.kt b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/LinearSearch.kt index 08daa082d0eb8f2ecbb71193111a0263ae275fbc..85deaf6fa75437199bfc560404eb5b40bb4a986a 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/LinearSearch.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/strategies/searchstrategy/LinearSearch.kt @@ -1,9 +1,12 @@ package theodolite.strategies.searchstrategy +import mu.KotlinLogging import theodolite.execution.BenchmarkExecutor import theodolite.util.LoadDimension import theodolite.util.Resource +private val logger = KotlinLogging.logger {} + /** * Linear-search-like implementation for determining the smallest suitable number of instances. * @@ -13,6 +16,8 @@ class LinearSearch(benchmarkExecutor: BenchmarkExecutor) : SearchStrategy(benchm override fun findSuitableResource(load: LoadDimension, resources: List<Resource>): Resource? { for (res in resources) { + + logger.info { "Running experiment with load '${load.get()}' and resources '${res.get()}'" } if (this.benchmarkExecutor.runExperiment(load, res)) return res } return null diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/util/DeploymentFailedException.kt b/theodolite-quarkus/src/main/kotlin/theodolite/util/DeploymentFailedException.kt new file mode 100644 index 0000000000000000000000000000000000000000..0e276d7de4e205a75eb309a71a793e70f7565ea4 --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/util/DeploymentFailedException.kt @@ -0,0 +1,5 @@ +package theodolite.util + + +class DeploymentFailedException(message:String): Exception(message) { +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/util/IOHandler.kt b/theodolite-quarkus/src/main/kotlin/theodolite/util/IOHandler.kt new file mode 100644 index 0000000000000000000000000000000000000000..8d379fcf0543257edafd2e45383a02ba0254563d --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/util/IOHandler.kt @@ -0,0 +1,94 @@ +package theodolite.util + +import com.google.gson.GsonBuilder +import mu.KotlinLogging +import java.io.File +import java.io.PrintWriter + +private val logger = KotlinLogging.logger {} + +/** + * The IOHandler handles most common I/O operations within the Theodolite framework + */ +class IOHandler { + + /** + * The location in which Theodolite store result and configuration file are depends on + * the values of the environment variables `RESULT_FOLDER` and `CREATE_RESULTS_FOLDER` + * + * @return the URL of the result folder + */ + fun getResultFolderURL(): String { + var resultsFolder: String = System.getenv("RESULTS_FOLDER") ?: "" + val createResultsFolder = System.getenv("CREATE_RESULTS_FOLDER") ?: "false" + + if (resultsFolder != ""){ + logger.info { "RESULT_FOLDER: $resultsFolder" } + val directory = File(resultsFolder) + if (!directory.exists()) { + logger.error { "Folder $resultsFolder does not exist" } + if (createResultsFolder.toBoolean()) { + directory.mkdirs() + } else { + throw IllegalArgumentException("Result folder not found") + } + } + resultsFolder += "/" + } + return resultsFolder + } + + /** + * Read a file as String + * + * @param fileURL the URL of the file + * @return The content of the file as String + */ + fun readFileAsString(fileURL: String): String { + return File(fileURL).inputStream().readBytes().toString(Charsets.UTF_8).trim() + } + + /** + * Creates a JSON string of the given object and store them to file + * + * @param T class of the object to save + * @param objectToSave object which should be saved as file + * @param fileURL the URL of the file + */ + fun <T> writeToJSONFile(objectToSave: T, fileURL: String) { + val gson = GsonBuilder().enableComplexMapKeySerialization().setPrettyPrinting().create() + writeStringToTextFile(fileURL, gson.toJson(objectToSave)) + } + + /** + * Write to CSV file + * + * @param fileURL the URL of the file + * @param data the data to write in the file, as list of list, each subList corresponds to a row in the CSV file + * @param columns columns of the CSV file + */ + fun writeToCSVFile(fileURL: String, data: List<List<String>>, columns: List<String>) { + val outputFile = File("$fileURL.csv") + PrintWriter(outputFile).use { pw -> + pw.println(columns.joinToString(separator=",")) + data.forEach { + pw.println(it.joinToString(separator=",")) + } + } + logger.info { "Wrote CSV file: $fileURL to ${outputFile.absolutePath}." } + } + + /** + * Write to text file + * + * @param fileURL the URL of the file + * @param data the data to write in the file as String + */ + fun writeStringToTextFile(fileURL: String, data: String) { + val outputFile = File("$fileURL") + outputFile.printWriter().use { + it.println(data) + } + logger.info { "Wrote txt file: $fileURL to ${outputFile.absolutePath}." } + } +} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt b/theodolite-quarkus/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt new file mode 100644 index 0000000000000000000000000000000000000000..c103ef1f35a1b3ffa56dad50c7cf6c1db51eb57f --- /dev/null +++ b/theodolite-quarkus/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt @@ -0,0 +1,5 @@ +package theodolite.util + +class InvalidPatcherConfigurationException(message:String): Exception(message) { +} + diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/util/KafkaConfig.kt b/theodolite-quarkus/src/main/kotlin/theodolite/util/KafkaConfig.kt index 4ba096e37345a9488cb288b21a8aa57ff07ac1ff..4e72ccb0d86749a6538c26556241ac114ef8d9a4 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/util/KafkaConfig.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/util/KafkaConfig.kt @@ -2,8 +2,9 @@ package theodolite.util import com.fasterxml.jackson.databind.annotation.JsonDeserialize import io.quarkus.runtime.annotations.RegisterForReflection -import org.apache.kafka.clients.admin.NewTopic +import theodolite.util.KafkaConfig.TopicWrapper import kotlin.properties.Delegates +import kotlin.reflect.KProperty /** * Configuration of Kafka connection. @@ -23,19 +24,11 @@ class KafkaConfig { */ lateinit var topics: List<TopicWrapper> - /** - * Get all current Kafka topics. - * - * @return the list of topics. - */ - fun getKafkaTopics(): List<NewTopic> { - return topics.map { topic -> NewTopic(topic.name, topic.numPartitions, topic.replicationFactor) } - } - /** * Wrapper for a topic definition. */ @RegisterForReflection + @JsonDeserialize class TopicWrapper { /** * The topic name @@ -51,5 +44,26 @@ class KafkaConfig { * The replication factor of this topic */ var replicationFactor by Delegates.notNull<Short>() + + /** + * If remove only, this topic would only used to delete all topics, which has the name of the topic as a prefix. + */ + var removeOnly by DelegatesFalse() + } +} + +/** + * Delegates to initialize a lateinit boolean to false + */ +@RegisterForReflection +class DelegatesFalse { + private var state = false + operator fun getValue(thisRef: Any?, property: KProperty<*>): Boolean { + return state + } + + operator fun setValue(thisRef: Any?, property: KProperty<*>, value: Boolean) { + state = value } + } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/util/PatcherDefinition.kt b/theodolite-quarkus/src/main/kotlin/theodolite/util/PatcherDefinition.kt index b24f887d6ff6e3096a2e740f541861d76804775b..6ec0cce36751ec0343d40aa49fefa44f4c7fc918 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/util/PatcherDefinition.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/util/PatcherDefinition.kt @@ -1,6 +1,7 @@ package theodolite.util import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import com.fasterxml.jackson.databind.annotation.JsonSerialize import io.quarkus.runtime.annotations.RegisterForReflection /** @@ -19,13 +20,6 @@ class PatcherDefinition { */ lateinit var resource: String - /** - * The container which the patcher is applied to - */ - lateinit var container: String - - /** - * The variable name for the patcher - */ - lateinit var variableName: String + @JsonSerialize + lateinit var properties: MutableMap<String, String> } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/util/PrometheusResponse.kt b/theodolite-quarkus/src/main/kotlin/theodolite/util/PrometheusResponse.kt index d1d59c482e64fd14c4744d8fcd606f286da24fb4..846577387c425e920da1c2fca1f972c880e1540a 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/util/PrometheusResponse.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/util/PrometheusResponse.kt @@ -1,6 +1,7 @@ package theodolite.util import io.quarkus.runtime.annotations.RegisterForReflection +import java.util.* /** * This class corresponds to the JSON response format of a Prometheus @@ -17,6 +18,27 @@ data class PrometheusResponse( */ var data: PromData? = null ) +{ + /** + * Return the data of the PrometheusResponse as [List] of [List]s of [String]s + * The format of the returned list is: `[[ group, timestamp, value ], [ group, timestamp, value ], ... ]` + */ + fun getResultAsList(): List<List<String>> { + val group = data?.result?.get(0)?.metric?.group.toString() + val values = data?.result?.get(0)?.values + val result = mutableListOf<List<String>>() + + if (values != null) { + for (value in values) { + val valueList = value as List<*> + val timestamp = (valueList[0] as Double).toLong().toString() + val value = valueList[1].toString() + result.add(listOf(group, timestamp, value)) + } + } + return Collections.unmodifiableList(result) + } +} /** * Description of Prometheus data. @@ -56,4 +78,5 @@ data class PromResult( @RegisterForReflection data class PromMetric( var group: String? = null -) \ No newline at end of file +) + diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/util/Results.kt b/theodolite-quarkus/src/main/kotlin/theodolite/util/Results.kt index 7116d73cf5b54325c8cfa41b1186d58695628874..60641ea0248435de53aaaaf362da7be995b391c5 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/util/Results.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/util/Results.kt @@ -40,23 +40,26 @@ class Results { * @param load the [LoadDimension] * * @return the smallest suitable number of resources. If the experiment was not executed yet, - * a @see Resource with the constant Int.MAX_VALUE as value is returned. If no experiments have been marked as either successful or unsuccessful + * a @see Resource with the constant Int.MAX_VALUE as value is returned. + * If no experiments have been marked as either successful or unsuccessful * yet, a Resource with the constant value Int.MIN_VALUE is returned. */ fun getMinRequiredInstances(load: LoadDimension?): Resource? { - if (this.results.isEmpty()) return Resource(Int.MIN_VALUE, emptyList()) + if (this.results.isEmpty()) { + return Resource(Int.MIN_VALUE, emptyList()) + } - var requiredInstances: Resource? = Resource(Int.MAX_VALUE, emptyList()) + var minRequiredInstances: Resource? = Resource(Int.MAX_VALUE, emptyList()) for (experiment in results) { + // Get all successful experiments for requested load if (experiment.key.first == load && experiment.value) { - if (requiredInstances == null) { - requiredInstances = experiment.key.second - } else if (experiment.key.second.get() < requiredInstances.get()) { - requiredInstances = experiment.key.second + if (minRequiredInstances == null || experiment.key.second.get() < minRequiredInstances.get()) { + // Found new smallest resources + minRequiredInstances = experiment.key.second } } } - return requiredInstances + return minRequiredInstances } /** @@ -70,13 +73,11 @@ class Results { fun getMaxBenchmarkedLoad(load: LoadDimension): LoadDimension? { var maxBenchmarkedLoad: LoadDimension? = null for (experiment in results) { - if (experiment.value) { - if (experiment.key.first.get() <= load.get()) { - if (maxBenchmarkedLoad == null) { - maxBenchmarkedLoad = experiment.key.first - } else if (maxBenchmarkedLoad.get() < experiment.key.first.get()) { - maxBenchmarkedLoad = experiment.key.first - } + if (experiment.key.first.get() <= load.get()) { + if (maxBenchmarkedLoad == null) { + maxBenchmarkedLoad = experiment.key.first + } else if (maxBenchmarkedLoad.get() < experiment.key.first.get()) { + maxBenchmarkedLoad = experiment.key.first } } } diff --git a/theodolite-quarkus/src/main/resources/operator/benchmarkCRD.yaml b/theodolite-quarkus/src/main/resources/operator/benchmarkCRD.yaml deleted file mode 100644 index 8fb3de1928f051d338a78ee58da074a73ef933c1..0000000000000000000000000000000000000000 --- a/theodolite-quarkus/src/main/resources/operator/benchmarkCRD.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: benchmarks.theodolite.com -spec: - group: theodolite.com - version: v1alpha1 - names: - kind: benchmark - plural: benchmarks - scope: Namespaced - subresources: - status: {} \ No newline at end of file diff --git a/theodolite-quarkus/src/main/resources/operator/example-execution-k8s-resource.yaml b/theodolite-quarkus/src/main/resources/operator/example-execution-k8s-resource.yaml index ef625dfe6ec78c2cc0ed099dfee0f767d57263bb..b81bbcd442834136283dc080f5f6a79bbc1cd415 100644 --- a/theodolite-quarkus/src/main/resources/operator/example-execution-k8s-resource.yaml +++ b/theodolite-quarkus/src/main/resources/operator/example-execution-k8s-resource.yaml @@ -1,37 +1,29 @@ -apiVersion: theodolite.com/v1alpha1 +apiVersion: theodolite.com/v1 kind: execution metadata: name: theodolite-example-execution -benchmark: "uc1-kstreams" -load: - loadType: "NumSensors" - loadValues: - - 50000 -resources: - resourceType: "Instances" - resourceValues: - - 1 -slos: - - sloType: "lag trend" - threshold: 1000 - prometheusUrl: "http://localhost:32656" - externalSloUrl: "http://localhost:80/evaluate-slope" - offset: 0 - warmup: 0 -execution: - strategy: "LinearSearch" - duration: 60 - repetitions: 1 - restrictions: - - "LowerBound" -configOverrides: - - patcher: - type: "NodeSelectorPatcher" - resource: "uc1-load-generator-deployment.yaml" - variableName: "env" - value: "prod" - - patcher: - type: "NodeSelectorPatcher" - resource: "uc1-kstreams-deployment.yaml" - variableName: "env" - value: "prod" \ No newline at end of file +spec: + benchmark: uc1-kstreams + load: + loadType: "NumSensors" + loadValues: + - 50000 + resources: + resourceType: "Instances" + resourceValues: + - 1 + slos: + - sloType: "lag trend" + threshold: 1000 + prometheusUrl: "http://localhost:32656" + externalSloUrl: "http://localhost:80/evaluate-slope" + offset: 0 + warmup: 0 + execution: + strategy: "LinearSearch" + duration: 60 + repetitions: 1 + loadGenerationDelay: 30 # in seconds + restrictions: + - "LowerBound" + configOverrides: [] \ No newline at end of file diff --git a/theodolite-quarkus/src/main/resources/operator/executionCRD.yaml b/theodolite-quarkus/src/main/resources/operator/executionCRD.yaml deleted file mode 100644 index 0bdb83c6201112a750bad41b81321b7a108a66fa..0000000000000000000000000000000000000000 --- a/theodolite-quarkus/src/main/resources/operator/executionCRD.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: executions.theodolite.com -spec: - group: theodolite.com - version: v1alpha1 - names: - kind: execution - plural: executions - scope: Namespaced - subresources: - status: {} \ No newline at end of file diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/CompositeStrategyTest.kt b/theodolite-quarkus/src/test/kotlin/theodolite/CompositeStrategyTest.kt index 7802529bfda309131cafc0ab3f39fda43285c32f..c2b30ab7eb23d60db39778218ad9d6a4c12799a6 100644 --- a/theodolite-quarkus/src/test/kotlin/theodolite/CompositeStrategyTest.kt +++ b/theodolite-quarkus/src/test/kotlin/theodolite/CompositeStrategyTest.kt @@ -31,7 +31,7 @@ class CompositeStrategyTest { val results = Results() val benchmark = TestBenchmark() val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo() - val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0) + val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 5) val linearSearch = LinearSearch(benchmarkExecutor) val lowerBoundRestriction = LowerBoundRestriction(results) val strategy = @@ -65,7 +65,7 @@ class CompositeStrategyTest { val benchmark = TestBenchmark() val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo() val benchmarkExecutorImpl = - TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0) + TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0, 0) val binarySearch = BinarySearch(benchmarkExecutorImpl) val lowerBoundRestriction = LowerBoundRestriction(results) val strategy = @@ -98,7 +98,7 @@ class CompositeStrategyTest { val results = Results() val benchmark = TestBenchmark() val sloChecker: BenchmarkExecution.Slo = BenchmarkExecution.Slo() - val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0) + val benchmarkExecutor = TestBenchmarkExecutorImpl(mockResults, benchmark, results, sloChecker, 0, 0,0) val binarySearch = BinarySearch(benchmarkExecutor) val lowerBoundRestriction = LowerBoundRestriction(results) val strategy = diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt b/theodolite-quarkus/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt index 82e4bc5d77f3f35d217c56a377513c0e7d329170..4ae9ce23b31204b366dce7bcf000e4114d7f3f8f 100644 --- a/theodolite-quarkus/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt +++ b/theodolite-quarkus/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt @@ -21,7 +21,7 @@ import theodolite.util.PatcherDefinition */ @QuarkusTest class ResourceLimitPatcherTest { - val testPath = "./src/main/resources/testYaml/" + val testPath = "./src/test/resources/" val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace("")) val patcherFactory = PatcherFactory() @@ -31,27 +31,32 @@ class ResourceLimitPatcherTest { val k8sResource = loader.loadK8sResource("Deployment", testPath + fileName) as Deployment val defCPU = PatcherDefinition() - defCPU.variableName = "cpu" defCPU.resource = "cpu-memory-deployment.yaml" - defCPU.container = "uc-application" defCPU.type = "ResourceLimitPatcher" + defCPU.properties = mutableMapOf( + "limitedResource" to "cpu", + "container" to "application" + ) val defMEM = PatcherDefinition() - defMEM.variableName = "memory" defMEM.resource = "cpu-memory-deployment.yaml" - defMEM.container = "uc-application" defMEM.type = "ResourceLimitPatcher" + defMEM.properties = mutableMapOf( + "limitedResource" to "memory", + "container" to "uc-application" + ) patcherFactory.createPatcher( patcherDefinition = defCPU, k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)) ).patch(value = cpuValue) + patcherFactory.createPatcher( patcherDefinition = defMEM, k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)) ).patch(value = memValue) - k8sResource.spec.template.spec.containers.filter { it.name == defCPU.container } + k8sResource.spec.template.spec.containers.filter { it.name == defCPU.properties["container"] !! } .forEach { assertTrue(it.resources.limits["cpu"].toString() == cpuValue) assertTrue(it.resources.limits["memory"].toString() == memValue) diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt b/theodolite-quarkus/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt index 3cd6b012f09c5471b1b011b5cd03e61a0fab1c4e..6f232961871c9d11929fc912a483e6e805cc7daf 100644 --- a/theodolite-quarkus/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt +++ b/theodolite-quarkus/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt @@ -21,7 +21,7 @@ import theodolite.util.PatcherDefinition */ @QuarkusTest class ResourceRequestPatcherTest { - val testPath = "./src/main/resources/testYaml/" + val testPath = "./src/test/resources/" val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace("")) val patcherFactory = PatcherFactory() @@ -31,16 +31,20 @@ class ResourceRequestPatcherTest { val k8sResource = loader.loadK8sResource("Deployment", testPath + fileName) as Deployment val defCPU = PatcherDefinition() - defCPU.variableName = "cpu" defCPU.resource = "cpu-memory-deployment.yaml" - defCPU.container = "uc-application" defCPU.type = "ResourceRequestPatcher" + defCPU.properties = mutableMapOf( + "requestedResource" to "cpu", + "container" to "application" + ) val defMEM = PatcherDefinition() - defMEM.variableName = "memory" defMEM.resource = "cpu-memory-deployment.yaml" - defMEM.container = "uc-application" defMEM.type = "ResourceRequestPatcher" + defMEM.properties = mutableMapOf( + "requestedResource" to "memory", + "container" to "application" + ) patcherFactory.createPatcher( patcherDefinition = defCPU, @@ -51,7 +55,7 @@ class ResourceRequestPatcherTest { k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)) ).patch(value = memValue) - k8sResource.spec.template.spec.containers.filter { it.name == defCPU.container } + k8sResource.spec.template.spec.containers.filter { it.name == defCPU.properties["container"] !! } .forEach { assertTrue(it.resources.requests["cpu"].toString() == cpuValue) assertTrue(it.resources.requests["memory"].toString() == memValue) diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt index 6f476278d08eacfc9857c1e5431636e5a219f26c..6ddca296acced658609fcccb307a83047238e118 100644 --- a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt +++ b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmark.kt @@ -11,7 +11,9 @@ class TestBenchmark : Benchmark { override fun buildDeployment( load: LoadDimension, res: Resource, - configurationOverrides: List<ConfigurationOverride?> + configurationOverrides: List<ConfigurationOverride?>, + loadGenerationDelay: Long, + afterTeardownDelay: Long ): BenchmarkDeployment { return TestBenchmarkDeployment() } diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt index 2bafcb76dfc3463d9aa350b88c9f73d52cea6629..cbd2d5926d61b0bfd4de6fab0c14422ddf88f190 100644 --- a/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt +++ b/theodolite-quarkus/src/test/kotlin/theodolite/TestBenchmarkExecutorImpl.kt @@ -13,7 +13,9 @@ class TestBenchmarkExecutorImpl( benchmark: Benchmark, results: Results, slo: BenchmarkExecution.Slo, - executionId: Int + executionId: Int, + loadGenerationDelay: Long, + afterTeardownDelay: Long ) : BenchmarkExecutor( benchmark, @@ -21,7 +23,10 @@ class TestBenchmarkExecutorImpl( executionDuration = Duration.ofSeconds(1), configurationOverrides = emptyList(), slo = slo, - executionId = executionId + repetitions = 1, + executionId = executionId, + loadGenerationDelay = loadGenerationDelay, + afterTeardownDelay = afterTeardownDelay ) { override fun runExperiment(load: LoadDimension, res: Resource): Boolean { diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/strategies/restriction/LowerBoundRestrictionTest.kt b/theodolite-quarkus/src/test/kotlin/theodolite/strategies/restriction/LowerBoundRestrictionTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..b368647e314a4d803b444268c8218aefbee00ad4 --- /dev/null +++ b/theodolite-quarkus/src/test/kotlin/theodolite/strategies/restriction/LowerBoundRestrictionTest.kt @@ -0,0 +1,118 @@ +package theodolite.strategies.restriction + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test +import theodolite.util.LoadDimension +import theodolite.util.Resource +import theodolite.util.Results + +internal class LowerBoundRestrictionTest { + + @Test + fun testNoPreviousResults() { + val results = Results() + val strategy = LowerBoundRestriction(results) + val load = buildLoadDimension(10000) + val resources = listOf( + buildResourcesDimension(1), + buildResourcesDimension(2), + buildResourcesDimension(3) + ) + val restriction = strategy.apply(load, resources) + + assertEquals(3, restriction.size) + assertEquals(resources, restriction) + } + + @Test + fun testWithSuccessfulPreviousResults() { + val results = Results() + results.setResult(10000, 1, true) + results.setResult(20000, 1, false) + results.setResult(20000, 2, true) + val strategy = LowerBoundRestriction(results) + val load = buildLoadDimension(30000) + val resources = listOf( + buildResourcesDimension(1), + buildResourcesDimension(2), + buildResourcesDimension(3) + ) + val restriction = strategy.apply(load, resources) + + assertEquals(2, restriction.size) + assertEquals(resources.subList(1, 3), restriction) + } + + @Test + @Disabled + fun testWithNoSuccessfulPreviousResults() { + // This test is currently not implemented this way, but might later be the desired behavior. + val results = Results() + results.setResult(10000, 1, true) + results.setResult(20000, 1, false) + results.setResult(20000, 2, false) + results.setResult(20000, 3, false) + val strategy = LowerBoundRestriction(results) + val load = buildLoadDimension(30000) + val resources = listOf( + buildResourcesDimension(1), + buildResourcesDimension(2), + buildResourcesDimension(3) + ) + val restriction = strategy.apply(load, resources) + + assertEquals(0, restriction.size) + assertEquals(emptyList<Resource>(), restriction) + } + + + @Test + fun testNoPreviousResults2() { + val results = Results() + results.setResult(10000, 1, true) + results.setResult(20000, 2, true) + results.setResult(10000, 1, false) + results.setResult(20000, 2, true) + + val minRequiredInstances = results.getMinRequiredInstances(LoadDimension(20000, emptyList())) + + assertNotNull(minRequiredInstances) + assertEquals(2, minRequiredInstances!!.get()) + } + + @Test + @Disabled + fun testMinRequiredInstancesWhenNotSuccessful() { + // This test is currently not implemented this way, but might later be the desired behavior. + val results = Results() + results.setResult(10000, 1, true) + results.setResult(20000, 2, true) + results.setResult(10000, 1, false) + results.setResult(20000, 2, false) + + val minRequiredInstances = results.getMinRequiredInstances(LoadDimension(20000, emptyList())) + + assertNotNull(minRequiredInstances) + assertEquals(2, minRequiredInstances!!.get()) + } + + private fun buildLoadDimension(load: Int): LoadDimension { + return LoadDimension(load, emptyList()) + } + + private fun buildResourcesDimension(resources: Int): Resource { + return Resource(resources, emptyList()) + } + + private fun Results.setResult(load: Int, resources: Int, successful: Boolean) { + this.setResult( + Pair( + buildLoadDimension(load), + buildResourcesDimension(resources) + ), + successful + ) + } +} diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/util/IOHandlerTest.kt b/theodolite-quarkus/src/test/kotlin/theodolite/util/IOHandlerTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..6984c3935da3d26f95565b758547a98e0eeb155f --- /dev/null +++ b/theodolite-quarkus/src/test/kotlin/theodolite/util/IOHandlerTest.kt @@ -0,0 +1,134 @@ +package theodolite.util + +import com.google.gson.GsonBuilder +import io.quarkus.test.junit.QuarkusTest +import org.hamcrest.CoreMatchers.containsString +import org.hamcrest.MatcherAssert.assertThat +import org.junit.Rule +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.junit.rules.TemporaryFolder +import org.junitpioneer.jupiter.ClearEnvironmentVariable +import org.junitpioneer.jupiter.SetEnvironmentVariable + + +const val FOLDER_URL = "Test-Folder" +@QuarkusTest +internal class IOHandlerTest { + + @Rule + private var temporaryFolder = TemporaryFolder() + + @Test + fun testWriteStringToText() { + temporaryFolder.create() + val testContent = "Test-File-Content" + val folder = temporaryFolder.newFolder(FOLDER_URL) + + IOHandler().writeStringToTextFile( + fileURL = "${folder.absolutePath}/test-file.txt", + data = testContent) + + assertEquals( + testContent, + IOHandler().readFileAsString("${folder.absolutePath}/test-file.txt") + ) + } + + @Test + fun testWriteToCSVFile() { + temporaryFolder.create() + val folder = temporaryFolder.newFolder(FOLDER_URL) + + val testContent = listOf( + listOf("apples","red"), + listOf("bananas","yellow"), + listOf("avocado","brown")) + val columns = listOf("Fruit", "Color") + + IOHandler().writeToCSVFile( + fileURL = "${folder.absolutePath}/test-file", + data = testContent, + columns = columns) + + var expected = "Fruit,Color\n" + testContent.forEach { expected += it[0] + "," + it[1] + "\n" } + + assertEquals( + expected.trim(), + IOHandler().readFileAsString("${folder.absolutePath}/test-file.csv") + ) + } + + @Test + fun testWriteToJSONFile() { + temporaryFolder.create() + val folder = temporaryFolder.newFolder(FOLDER_URL) + val testContent = Resource(0, emptyList()) + + IOHandler().writeToJSONFile( + fileURL = "${folder.absolutePath}/test-file.json", + objectToSave = testContent) + + val expected = GsonBuilder().enableComplexMapKeySerialization().setPrettyPrinting().create().toJson(testContent) + + assertEquals( + expected, + IOHandler().readFileAsString("${folder.absolutePath}/test-file.json") + ) + } + + // Test the function `getResultFolderString` + + @Test + @ClearEnvironmentVariable.ClearEnvironmentVariables( + ClearEnvironmentVariable(key = "RESULTS_FOLDER"), + ClearEnvironmentVariable(key = "CREATE_RESULTS_FOLDER") + ) + fun testGetResultFolderURL_emptyEnvironmentVars() { + assertEquals("",IOHandler().getResultFolderURL()) + } + + + @Test() + @SetEnvironmentVariable.SetEnvironmentVariables( + SetEnvironmentVariable(key = "RESULTS_FOLDER", value = "./src/test/resources"), + SetEnvironmentVariable(key = "CREATE_RESULTS_FOLDER", value = "false") + ) + fun testGetResultFolderURL_FolderExist() { + assertEquals("./src/test/resources/", IOHandler().getResultFolderURL()) + } + + @Test() + @SetEnvironmentVariable.SetEnvironmentVariables( + SetEnvironmentVariable(key = "RESULTS_FOLDER", value = "$FOLDER_URL-0"), + SetEnvironmentVariable(key = "CREATE_RESULTS_FOLDER", value = "false") + ) + fun testGetResultFolderURL_FolderNotExist() { + var exceptionWasThrown = false + try { + IOHandler().getResultFolderURL() + } catch (e: Exception){ + exceptionWasThrown = true + assertThat(e.toString(), containsString("Result folder not found")); + } + assertTrue(exceptionWasThrown) + } + + @Test() + @SetEnvironmentVariable.SetEnvironmentVariables( + SetEnvironmentVariable(key = "RESULTS_FOLDER", value = FOLDER_URL), + SetEnvironmentVariable(key = "CREATE_RESULTS_FOLDER", value = "true") + ) + fun testGetResultFolderURL_CreateFolderIfNotExist() { + assertEquals("$FOLDER_URL/", IOHandler().getResultFolderURL()) + } + + @Test() + @ClearEnvironmentVariable(key = "RESULTS_FOLDER" ) + @SetEnvironmentVariable(key = "CREATE_RESULTS_FOLDER", value = "true") + fun testGetResultFolderURL_CreateFolderButNoFolderGiven() { + assertEquals("", IOHandler().getResultFolderURL()) + } +} diff --git a/theodolite-quarkus/src/test/kotlin/theodolite/util/ResultsTest.kt b/theodolite-quarkus/src/test/kotlin/theodolite/util/ResultsTest.kt new file mode 100644 index 0000000000000000000000000000000000000000..9cfc2ae78e7a8846e3f0fa136699509145e5de22 --- /dev/null +++ b/theodolite-quarkus/src/test/kotlin/theodolite/util/ResultsTest.kt @@ -0,0 +1,75 @@ +package theodolite.util + +import io.quarkus.test.junit.QuarkusTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test + +@QuarkusTest +internal class ResultsTest { + + @Test + fun testMinRequiredInstancesWhenSuccessful() { + val results = Results() + results.setResult(10000, 1, true) + results.setResult(10000, 2, true) + results.setResult(20000, 1, false) + results.setResult(20000, 2, true) + + val minRequiredInstances = results.getMinRequiredInstances(LoadDimension(20000, emptyList())) + + assertNotNull(minRequiredInstances) + assertEquals(2, minRequiredInstances!!.get()) + } + + @Test + @Disabled + fun testMinRequiredInstancesWhenNotSuccessful() { + // This test is currently not implemented this way, but might later be the desired behavior. + val results = Results() + results.setResult(10000, 1, true) + results.setResult(10000, 2, true) + results.setResult(20000, 1, false) + results.setResult(20000, 2, false) + + val minRequiredInstances = results.getMinRequiredInstances(LoadDimension(20000, emptyList())) + + assertNotNull(minRequiredInstances) + assertEquals(2, minRequiredInstances!!.get()) + } + + private fun Results.setResult(load: Int, resources: Int, successful: Boolean) { + this.setResult( + Pair( + LoadDimension(load, emptyList()), + Resource(resources, emptyList()) + ), + successful + ) + } + + + @Test + fun testGetMaxBenchmarkedLoadWhenAllSuccessful() { + val results = Results() + results.setResult(10000, 1, true) + results.setResult(10000, 2, true) + + val test1 = results.getMaxBenchmarkedLoad(LoadDimension(100000, emptyList()))!!.get() + + assertEquals(10000, test1) + } + + @Test + fun testGetMaxBenchmarkedLoadWhenLargestNotSuccessful() { + val results = Results() + results.setResult(10000, 1, true) + results.setResult(10000, 2, true) + results.setResult(20000, 1, false) + + val test2 = results.getMaxBenchmarkedLoad(LoadDimension(100000, emptyList()))!!.get() + + assertEquals(20000, test2) + } +} diff --git a/theodolite-quarkus/src/main/resources/testYaml/cpu-deployment.yaml b/theodolite-quarkus/src/test/resources/cpu-deployment.yaml similarity index 100% rename from theodolite-quarkus/src/main/resources/testYaml/cpu-deployment.yaml rename to theodolite-quarkus/src/test/resources/cpu-deployment.yaml diff --git a/theodolite-quarkus/src/main/resources/testYaml/cpu-memory-deployment.yaml b/theodolite-quarkus/src/test/resources/cpu-memory-deployment.yaml similarity index 100% rename from theodolite-quarkus/src/main/resources/testYaml/cpu-memory-deployment.yaml rename to theodolite-quarkus/src/test/resources/cpu-memory-deployment.yaml diff --git a/theodolite-quarkus/src/main/resources/testYaml/memory-deployment.yaml b/theodolite-quarkus/src/test/resources/memory-deployment.yaml similarity index 100% rename from theodolite-quarkus/src/main/resources/testYaml/memory-deployment.yaml rename to theodolite-quarkus/src/test/resources/memory-deployment.yaml diff --git a/theodolite-quarkus/src/main/resources/testYaml/no-resources-deployment.yaml b/theodolite-quarkus/src/test/resources/no-resources-deployment.yaml similarity index 100% rename from theodolite-quarkus/src/main/resources/testYaml/no-resources-deployment.yaml rename to theodolite-quarkus/src/test/resources/no-resources-deployment.yaml