diff --git a/.gitignore b/.gitignore index 71305e60a1056e58f281da4c2ab397539b63ba52..e898f39a2d997d01680ff63d1fe375b4b83b00b7 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ tmp/ # Python cache files *.pyc + +# Helm +Chart.lock diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4ee62b4c634b8833749f63dc583e5659809dd7d6..15d29f4e47319043954f923e4840c0ecd7dcbcca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,17 @@ stages: variables: DOCKER_TLS_CERTDIR: "/certs" +# Theodolite Helm Chart + +lint-helm: + stage: check + image: + name: alpine/helm:3.5.2 + entrypoint: [""] + tags: + - exec-docker + script: helm lint execution/helm/ + # Theodolite Benchmarks diff --git a/docs/release-process.md b/docs/release-process.md index 2a679b90c226efede22634970039c76ead1b6418..981306b0762e43eacb29a434cc1e505593548fce 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -11,22 +11,42 @@ This document describes how to perform a new Theodolite release. We assume that we are creating the release `v0.3.1`. Please make sure to adjust the following steps according to the release, you are actually performing. -1. Update `codemeta.json` to match the new version. In particular, make sure that `version` points to the version you are releasing and `dateModified` points to the date you are relasing this version. [CodeMeata generator](https://codemeta.github.io/codemeta-generator/) may help you in updating the file. +1. Create a new branch `v0.3` if it does not already exist. This branch will never +again be merged into master. -2. Update `CITATION.cff` to match the new version. At least update the `version` field. +2. Checkout the `v0.3` branch. -3. Commit these changes to the `master` branch. +3. Update all references to artifacts which are versioned. This includes: -4. Create a new branch `v0.3` if it does not already exists. This branch will never -again be merged into master. + 1. Update all references to Theodolite Docker images to tag `v0.3.1`. These are the Kubernetes resource definitions in +`execution`, the references to *latest* in `run_uc.py`, the Docker Compose files in `theodolite-benchmarks/docker-test` and the example `theodolite.yaml` job. + + 2. Update both, the `version` and the `appVersion` fields, in the Helm `Charts.yaml` file to `0.3.1`. + + 3. Update `codemeta.json` to match the new version. In particular, make sure that `version` points to the version you are releasing and `dateModified` points to the date you are relasing this version. [CodeMeata generator](https://codemeta.github.io/codemeta-generator/) may help you in updating the file. + + 4. Update `CITATION.cff` to match the new version. At least update the `version` field. + +4. Create a Helm package by running `./build-package.sh` from the chart directory. + +5. Update the Helm repository index of located at `/docs` by running `./update-index.sh v0.3.1`. + +6. Commit these changes to the `v0.3` branch. + +7. Tag this commit `v0.3.1` (can be done via GitLab). The corresponding Docker images will be uploaded. + +8. Create *releases* on GitLab and GitHub. Upload the generated Helm package to these releases. + +9. Switch to the `master` branch. + +10. Re-run `./update-index.sh v0.3.1` to include the latest release in the *upstream* Helm repository. You can now delete the packaged Helm chart. -5. Checkout the `v0.3` branch. +11. If this release increments Theodolite's *latest* version number, -6. Update all references to Theodolite Docker images to tag `v0.3.1`. These are the Kubernetes resource definitions in -`execution`, the references to *latest* in `run_uc.py`, the Docker Compose files in `docker-test` and the example `theodolite.yaml` job. + 1. Update the Helm `Charts.yaml` file to `0.4.0-SNAPSHOT` (see Step 3). -7. Commit these changes. + 2. Update the `codemeta.json` file according to Step 3. -8. Tag this commit with `v0.3.1`. The corresponding Docker images will be uploaded. + 3. Update the `CITATION.cff` file according to Step 3. -9. Create *releases* for this tag in both, GitLab and GitHub. +12. Commit these changes to the `master` branch. diff --git a/execution/helm/.helmignore b/execution/helm/.helmignore new file mode 100644 index 0000000000000000000000000000000000000000..0e8a0eb36f4ca2c939201c0d54b5d82a1ea34778 --- /dev/null +++ b/execution/helm/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/execution/helm/Chart.yaml b/execution/helm/Chart.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d4b32b2c63555c2d91f38497aa2317f97e68bc6 --- /dev/null +++ b/execution/helm/Chart.yaml @@ -0,0 +1,34 @@ +apiVersion: v2 +name: theodolite +description: Theodolite is a framework for benchmarking the scalability stream processing engines. +home: https://cau-se.github.io/theodolite +sources: + - https://github.com/cau-se/theodolite +maintainers: +- name: Sören Henning + email: soeren.henning@email.uni-kiel.de + url: https://www.se.informatik.uni-kiel.de/en/team/soeren-henning-m-sc + +type: application + +dependencies: + - name: grafana + version: 6.0.0 + repository: https://grafana.github.io/helm-charts + condition: grafana.enabled + - name: kube-prometheus-stack + version: 12.0.0 + repository: https://prometheus-community.github.io/helm-charts + condition: kube-prometheus-stack.enabled + - name: cp-helm-charts + version: 0.6.0 + repository: https://soerenhenning.github.io/cp-helm-charts + condition: cp-helm-charts.enabled + - name: kafka-lag-exporter + version: 0.6.6 + repository: https://lightbend.github.io/kafka-lag-exporter/repo/ + condition: kafka-lag-exporter.enabled + +version: 0.4.0-SNAPSHOT + +appVersion: 0.4.0-SNAPSHOT diff --git a/execution/helm/README.md b/execution/helm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4cacd06c8181970e78cb4f62e93b77fa169fcdfa --- /dev/null +++ b/execution/helm/README.md @@ -0,0 +1,54 @@ +# Theodolite Helm Chart + +## Installation + +Install the chart via: + +```sh +helm dependencies update . +helm install my-confluent . +``` + +**Please note: Theodolite currently uses hard-coded URLs, to connect to Kafka and Zookeeper. For that reason, the name of this chart must be `my-confluent`.** We will change this behavior soon. + +This chart installs requirements to execute benchmarks with Theodolite. + +Dependencies and subcharts: + +- Prometheus Operator +- Prometheus +- Grafana (incl. dashboard and data source configuration) +- Kafka +- Zookeeper +- A Kafka client pod + +## Test + +Test the installation: + +```sh +helm test <release-name> +``` + +Our test files are located [here](templates/../../theodolite-chart/templates/tests). Many subcharts have their own tests, these are also executed and are placed in the respective /templates folders. + +Please note: If a test fails, Helm will stop testing. + +It is possible that the tests are not running successfully at the moment. This is because the Helm tests of the subchart cp-confluent receive a timeout exception. There is an [issue](https://github.com/confluentinc/cp-helm-charts/issues/318) for this problem on GitHub. + +## Configuration + +In development environments Kubernetes resources are often low. To reduce resource consumption, we provide an `one-broker-value.yaml` file. This file can be used with: + +```sh +helm install theodolite . -f preconfigs/one-broker-values.yaml +``` + +## Development + +**Hints**: + +- Grafana configuration: Grafana ConfigMaps contains expressions like {{ topic }}. Helm uses the same syntax for template function. More information [here](https://github.com/helm/helm/issues/2798) + - Escape braces: {{ "{{" topic }} + - Let Helm render the template as raw string: {{ `{{ <config>}}` }} + \ No newline at end of file diff --git a/execution/helm/build-package.sh b/execution/helm/build-package.sh new file mode 100755 index 0000000000000000000000000000000000000000..e79d0497d883a8e1e0fab56ddeeb8d4ee1053648 --- /dev/null +++ b/execution/helm/build-package.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +helm package . --dependency-update && rm -r charts # We don't want to include dependencies in our index diff --git a/execution/helm/preconfigs/one-broker-values.yaml b/execution/helm/preconfigs/one-broker-values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fdbc3207ee37f49cf176645851d91e62ba354d28 --- /dev/null +++ b/execution/helm/preconfigs/one-broker-values.yaml @@ -0,0 +1,15 @@ +cp-helm-charts: + ## ------------------------------------------------------ + ## Zookeeper + ## ------------------------------------------------------ + cp-zookeeper: + servers: 1 # default: 3 + + ## ------------------------------------------------------ + ## Kafka + ## ------------------------------------------------------ + cp-kafka: + brokers: 1 # deauflt: 10 + + configurationOverrides: + offsets.topic.replication.factor: "1" \ No newline at end of file diff --git a/execution/helm/templates/NOTES.txt b/execution/helm/templates/NOTES.txt new file mode 100644 index 0000000000000000000000000000000000000000..ef1eea71080f55d08e193b9741327189865fa3dd --- /dev/null +++ b/execution/helm/templates/NOTES.txt @@ -0,0 +1,3 @@ +Welcome to Theodolite! + +Visit https://cau-se.github.io/theodolite for getting started and more information. diff --git a/execution/helm/templates/_helpers.tpl b/execution/helm/templates/_helpers.tpl new file mode 100644 index 0000000000000000000000000000000000000000..f59f74d369b64ec89a44cbf2048fda9e844df92b --- /dev/null +++ b/execution/helm/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "theodolite.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "theodolite.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "theodolite.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "theodolite.labels" -}} +helm.sh/chart: {{ include "theodolite.chart" . }} +{{ include "theodolite.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "theodolite.selectorLabels" -}} +app.kubernetes.io/name: {{ include "theodolite.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "theodolite.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "theodolite.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/execution/helm/templates/cluster-role-binding.yaml b/execution/helm/templates/cluster-role-binding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..400a972cdb73dca181b621f49e7a3e79c926e65b --- /dev/null +++ b/execution/helm/templates/cluster-role-binding.yaml @@ -0,0 +1,14 @@ +{{- if .Values.prometheus.clusterRoleBinding.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: {{ .Release.Namespace }} +{{- end}} \ No newline at end of file diff --git a/execution/helm/templates/cluster-role.yaml b/execution/helm/templates/cluster-role.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a272718da1413a466d6afed51b3bca1f37a1fe0 --- /dev/null +++ b/execution/helm/templates/cluster-role.yaml @@ -0,0 +1,20 @@ +{{- if .Values.prometheus.clusterRole.enabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} \ No newline at end of file diff --git a/execution/helm/templates/dashboard-config-map.yaml b/execution/helm/templates/dashboard-config-map.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87e588f29df7446d0b12000eb53487a9bb88ea6c --- /dev/null +++ b/execution/helm/templates/dashboard-config-map.yaml @@ -0,0 +1,1007 @@ +{{- if .Values.grafana.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: scalability + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + {{`{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 2, + "iteration": 1589140028684, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_input)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{Messages In Per Second}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages In Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(cp_kafka_server_brokertopicmetrics_messagesinpersec_topic_output)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{Messages Out Per Second}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages Out Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(group, topic) (kafka_consumergroup_group_lag > 0)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Record Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count(count (kafka_consumer_consumer_fetch_manager_metrics_records_lag) by(pod))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "instances", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Instances", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(group,topic) (kafka_consumergroup_group_offset > 0)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Consumed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count by(group,topic) (kafka_consumergroup_group_offset > 0)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Partitions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(group,topic) (kafka_partition_latest_offset)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Produced (Kafka Lag Exporter)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count by(job, topic) (kafka_consumer_consumer_fetch_manager_metrics_records_lag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Partitions (Kafka Streams Export)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(job, topic) (kafka_consumer_consumer_fetch_manager_metrics_records_lag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Record Lag (Kafka Streams Export)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(group) (kafka_consumergroup_group_lag > 0)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "total lag", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Record Lag (Kafka Lag Exporter)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 21, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "titan-ccp-aggregation", + "value": "titan-ccp-aggregation" + }, + "datasource": "Prometheus", + "definition": "label_values(kafka_consumer_consumer_fetch_manager_metrics_records_lag, job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "Job", + "options": [ + { + "selected": true, + "text": "titan-ccp-aggregation", + "value": "titan-ccp-aggregation" + } + ], + "query": "label_values(kafka_consumer_consumer_fetch_manager_metrics_records_lag, job)", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Scalability Benchmarking", + "uid": "dad0CNlZz", + "version": 25 + }`}} +{{- end }} \ No newline at end of file diff --git a/execution/helm/templates/kafka-client.yaml b/execution/helm/templates/kafka-client.yaml new file mode 100644 index 0000000000000000000000000000000000000000..853cb768672d8888085a3881df81cbdb806ec39d --- /dev/null +++ b/execution/helm/templates/kafka-client.yaml @@ -0,0 +1,19 @@ +{{- if .Values.kafkaClient.enabled -}} +apiVersion: v1 +kind: Pod +metadata: + # name: {{ template "theodolite.fullname" . }}-kafka-client + name: kafka-client +spec: + containers: + - name: kafka-client + image: confluentinc/cp-enterprise-kafka:5.4.0 + command: + - sh + - -c + - "exec tail -f /dev/null" + {{- with .Values.kafkaClient.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/execution/helm/templates/prometheus-datasource-config-map.yaml b/execution/helm/templates/prometheus-datasource-config-map.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e793ff83668ac7a7582a924750ca729d9e277ae --- /dev/null +++ b/execution/helm/templates/prometheus-datasource-config-map.yaml @@ -0,0 +1,31 @@ +{{- if .Values.grafana.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus + labels: + grafana_datasource: "1" +data: + datasource.yaml: |- + # config file version + apiVersion: 1 + datasources: + # <string, required> name of the datasource. Required + - name: Prometheus + # <string, required> datasource type. Required + type: prometheus + # <string, required> access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # <bool> mark as default datasource. Max one per org + isDefault: true + # <int> org id. will default to orgId 1 if not specified + orgId: 1 + # <string> url + url: http://prometheus-operated:9090 #http://localhost:9090 + # <map> fields that will be converted to json and stored in json_data + jsonData: + timeInterval: "15s" + version: 1 + # <bool> allow users to edit datasources from the UI. + editable: true +{{- end }} \ No newline at end of file diff --git a/execution/helm/templates/prometheus.yaml b/execution/helm/templates/prometheus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3060798a8a3b000f730525805c0d050becc7a68 --- /dev/null +++ b/execution/helm/templates/prometheus.yaml @@ -0,0 +1,21 @@ +{{- if .Values.prometheus.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: {{ template "theodolite.fullname" . }}-prometheus +spec: + serviceAccountName: prometheus + serviceMonitorSelector: + matchLabels: + #app: cp-kafka + appScope: titan-ccp + resources: + requests: + memory: 400Mi + #scrapeInterval: 1s + enableAdminAPI: true + {{- with .Values.prometheus.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end}} +{{- end}} \ No newline at end of file diff --git a/execution/helm/templates/service-account.yaml b/execution/helm/templates/service-account.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e14c8eb8ffd912f3d34d1b94aa481cb497b4b90 --- /dev/null +++ b/execution/helm/templates/service-account.yaml @@ -0,0 +1,6 @@ +{{- if .Values.prometheus.serviceAccount.enabled -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus +{{- end}} \ No newline at end of file diff --git a/execution/helm/templates/service-monitor.yaml b/execution/helm/templates/service-monitor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..50e4688cf7f9b919afdc9455462034f682975893 --- /dev/null +++ b/execution/helm/templates/service-monitor.yaml @@ -0,0 +1,16 @@ +{{- if .Values.kafkaClient.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app: cp-kafka + appScope: titan-ccp + name: kafka +spec: + selector: + matchLabels: + app: cp-kafka + endpoints: + - port: metrics + interval: 7s +{{- end}} diff --git a/execution/helm/templates/tests/test-connection.yaml b/execution/helm/templates/tests/test-connection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7af87e98920c11bcfaccb27724e6f29fc76771a0 --- /dev/null +++ b/execution/helm/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "theodolite.fullname" . }}-test-prometheus" + labels: + {{- include "theodolite.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['http://prometheus-operated:9090'] + restartPolicy: Never diff --git a/execution/helm/update-index.sh b/execution/helm/update-index.sh new file mode 100755 index 0000000000000000000000000000000000000000..286724dd87718387df58ed993af417bf0fd4d8ec --- /dev/null +++ b/execution/helm/update-index.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env sh + +RELEASE_NAME=$1 # Supposed to be equal to tag, e.g., v0.3.0 + +RELEASE_PATH="https://github.com/cau-se/theodolite/releases/download" +REPO_INDEX="../../docs/index.yaml" + +helm repo index . --url $RELEASE_PATH/$RELEASE_NAME --merge $REPO_INDEX && \ + mv index.yaml $REPO_INDEX \ No newline at end of file diff --git a/execution/helm/values.yaml b/execution/helm/values.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d35912de6ffc72fff27f5d389c94761bc80eecd1 --- /dev/null +++ b/execution/helm/values.yaml @@ -0,0 +1,229 @@ +### +# Theodolite resources +### + +kafkaClient: + enabled: true + nodeSelector: {} + + +#### +## configuration of sub charts +### + +### +# Grafana +### +grafana: + enabled: true + nodeSelector: {} + image: + repository: grafana/grafana + tag: 6.7.3 + pullPolicy: IfNotPresent + # Administrator credentials when not using an existing secret (see below) + adminUser: admin + adminPassword: admin + ## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders + ## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards + sidecar: + image: + repository: "kiwigrid/k8s-sidecar" + tag: "0.1.99" + imagePullPolicy: IfNotPresent + dashboards: + enabled: true + provider: + # allow updating provisioned dashboards from the UI + allowUiUpdates: true + datasources: + enabled: true + service: + nodePort: 31199 + type: NodePort + + +### +# Confluent Platform +### + +cp-helm-charts: + enabled: true + ## ------------------------------------------------------ + ## Zookeeper + ## ------------------------------------------------------ + cp-zookeeper: + enabled: true + nodeSelector: {} + servers: 3 # default: 3 + image: confluentinc/cp-zookeeper + imageTag: 5.4.0 + ## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. + ## https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod + imagePullSecrets: + # - name: "regcred" + heapOptions: "-Xms512M -Xmx512M" + persistence: + enabled: false + + ## ------------------------------------------------------ + ## Kafka + ## ------------------------------------------------------ + cp-kafka: + enabled: true + nodeSelector: {} + brokers: 10 # default: 10 + image: confluentinc/cp-enterprise-kafka + imageTag: 5.4.0 + ## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. + ## https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod + imagePullSecrets: + # - name: "regcred" + heapOptions: "-Xms512M -Xmx512M" + persistence: + enabled: false + resources: {} + configurationOverrides: + #offsets.topic.replication.factor: 1 + "message.max.bytes": "134217728" # 128 MB + "replica.fetch.max.bytes": "134217728" # 128 MB + #default.replication.factor: 1 + # "min.insync.replicas": 2 + # "auto.create.topics.enable": false + "log.retention.ms": "10000" # 10s + "metrics.sample.window.ms": "5000" #5s + "advertised.listeners": |- + EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID})) + "listener.security.protocol.map": |- + PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT + + nodeport: + enabled: false + servicePort: 19092 + firstListenerPort: 31090 + + + ## ------------------------------------------------------ + ## Schema Registry + ## ------------------------------------------------------ + cp-schema-registry: + enabled: true + nodeSelector: {} + image: confluentinc/cp-schema-registry + imageTag: 5.4.0 + ## Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. + ## https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod + imagePullSecrets: + # - name: "regcred" + heapOptions: "-Xms512M -Xmx512M" + resources: {} + + external: + enabled: true + type: NodePort + servicePort: 8081 + nodePort: 30099 + annotations: {} + + cp-kafka-rest: + enabled: false + + cp-kafka-connect: + enabled: false + + cp-ksql-server: + enabled: false + + cp-control-center: + enabled: false + + +### +# Kafka Lag Exporter +### +kafka-lag-exporter: + enabled: true + nodeSelector: {} + clusters: + - name: "my-confluent-cp-kafka" + bootstrapBrokers: "my-confluent-cp-kafka:9092" + + ## The interval between refreshing metrics + pollIntervalSeconds: 15 + + prometheus: + serviceMonitor: + enabled: true + interval: "5s" + additionalLabels: + appScope: titan-ccp + + +### +# Prometheus Monitoring Stack (Prometheus Operator) +### +kube-prometheus-stack: + commonLabels: + appScope: titan-ccp + + alertmanager: + enabled: false + + grafana: + enabled: false + + kubeApiServer: + enabled: false + + kubelet: + enabled: false + + kubeControllerManager: + enabled: false + + coreDns: + enabled: false + + kubeDns: + enabled: false + + kubeEtcd: + enabled: false + + kubeScheduler: + enabled: false + + kubeProxy: + enabled: false + + kubeStateMetrics: + enabled: false + + nodeExporter: + enabled: false + + prometheusOperator: + enabled: true + namespaces: + releaseNamespace: true + additional: [] + nodeSelector: {} + + prometheus: + enabled: false + + +### +# Prometheus +### +prometheus: + enabled: true + nodeSelector: {} + + # depends on your cluster security and permission settings, you may need to create the following resources + serviceAccount: + enabled: true + clusterRole: + enabled: true + clusterRoleBinding: + enabled: true \ No newline at end of file