Merge branch '0.5-docs' into 'master'

Add Theodolite docs

Closes #181 and #237

See merge request !164

.gitlab-ci.yml

@@ -41,6 +41,60 @@ default:
     - /kaniko/executor --context `pwd`/$CONTEXT $KANIKO_DOCKERFILE $KANIKO_D
 
 
+# Theodolite Docs
+
+.docs:
+  image: alpine/bundle:3.0.3
+  cache:
+    paths:
+      - docs/vendor
+  before_script:
+    - cd docs
+    - bundle config set --local path 'vendor'
+    - bundle install
+  rules:
+    - changes:
+      - docs/**/*
+    - when: manual
+      allow_failure: true
+
+build-docs:
+  stage: build
+  extends: .docs
+  script: bundle exec jekyll build
+  artifacts:
+    paths:
+      - docs/_site
+
+test-docs-links:
+  stage: test
+  extends: .docs
+  needs:
+    - build-docs
+  script: bundle exec htmlproofer --assume-extension --allow_hash_href --url-ignore "/favicon.ico" ./_site
+
+test-docs-crds-regression:
+  stage: test
+  image: golang
+  before_script:
+    - cd docs
+    - go install fybrik.io/crdoc@latest
+  script:
+    - crdoc --resources ../theodolite/crd/ --template api-reference/crds.tmpl  --output api-reference/crds.ref.md
+    - cmp api-reference/crds.md api-reference/crds.ref.md
+  artifacts:
+    when: on_failure
+    paths:
+      - docs/api-reference/crds.ref.md
+    expire_in: 1 week
+  rules:
+    - changes:
+      - docs/api-reference/crds.tmpl
+      - theodolite/crd/**/*
+    - when: manual
+      allow_failure: true
+
+
 # Theodolite Helm Chart
 
 lint-helm:

docs/.gitignore

+_site
+.sass-cache
+.jekyll-cache
+.jekyll-metadata
+vendor

docs/Gemfile

+source "https://rubygems.org"
+# Hello! This is where you manage which Jekyll version is used to run.
+# When you want to use a different version, change it below, save the
+# file and run `bundle install`. Run Jekyll with `bundle exec`, like so:
+#
+#     bundle exec jekyll serve
+#
+# This will help ensure the proper Jekyll version is running.
+# Happy Jekylling!
+#gem "jekyll", "~> 4.2.0"
+# This is the default theme for new Jekyll sites. You may change this to anything you like.
+gem "minima", "~> 2.5"
+# If you want to use GitHub Pages, remove the "gem "jekyll"" above and
+# uncomment the line below. To upgrade, run `bundle update github-pages`.
+# gem "github-pages", group: :jekyll_plugins
+gem "github-pages", "~> 215", group: :jekyll_plugins
+# If you have any plugins, put them here!
+#group :jekyll_plugins do
+  #gem "jekyll-feed", "~> 0.12"
+#end
+
+gem 'html-proofer'
+
+# Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem
+# and associated library.
+platforms :mingw, :x64_mingw, :mswin, :jruby do
+  gem "tzinfo", "~> 1.2"
+  gem "tzinfo-data"
+end
+
+# Performance-booster for watching directories on Windows
+gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin]
+

docs/Gemfile.lock

+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.0.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.2, >= 2.2.2)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    coffee-script (2.4.1)
+      coffee-script-source
+      execjs
+    coffee-script-source (1.11.1)
+    colorator (1.1.0)
+    commonmarker (0.17.13)
+      ruby-enum (~> 0.5)
+    concurrent-ruby (1.1.9)
+    dnsruby (1.61.7)
+      simpleidn (~> 0.1)
+    em-websocket (0.5.2)
+      eventmachine (>= 0.12.9)
+      http_parser.rb (~> 0.6.0)
+    ethon (0.14.0)
+      ffi (>= 1.15.0)
+    eventmachine (1.2.7)
+    execjs (2.8.1)
+    faraday (1.4.3)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.1.0)
+    ffi (1.15.3)
+    forwardable-extended (2.6.0)
+    gemoji (3.0.1)
+    github-pages (215)
+      github-pages-health-check (= 1.17.2)
+      jekyll (= 3.9.0)
+      jekyll-avatar (= 0.7.0)
+      jekyll-coffeescript (= 1.1.1)
+      jekyll-commonmark-ghpages (= 0.1.6)
+      jekyll-default-layout (= 0.1.4)
+      jekyll-feed (= 0.15.1)
+      jekyll-gist (= 1.5.0)
+      jekyll-github-metadata (= 2.13.0)
+      jekyll-mentions (= 1.6.0)
+      jekyll-optional-front-matter (= 0.3.2)
+      jekyll-paginate (= 1.1.0)
+      jekyll-readme-index (= 0.3.0)
+      jekyll-redirect-from (= 0.16.0)
+      jekyll-relative-links (= 0.6.1)
+      jekyll-remote-theme (= 0.4.3)
+      jekyll-sass-converter (= 1.5.2)
+      jekyll-seo-tag (= 2.7.1)
+      jekyll-sitemap (= 1.4.0)
+      jekyll-swiss (= 1.0.0)
+      jekyll-theme-architect (= 0.1.1)
+      jekyll-theme-cayman (= 0.1.1)
+      jekyll-theme-dinky (= 0.1.1)
+      jekyll-theme-hacker (= 0.1.2)
+      jekyll-theme-leap-day (= 0.1.1)
+      jekyll-theme-merlot (= 0.1.1)
+      jekyll-theme-midnight (= 0.1.1)
+      jekyll-theme-minimal (= 0.1.1)
+      jekyll-theme-modernist (= 0.1.1)
+      jekyll-theme-primer (= 0.5.4)
+      jekyll-theme-slate (= 0.1.1)
+      jekyll-theme-tactile (= 0.1.1)
+      jekyll-theme-time-machine (= 0.1.1)
+      jekyll-titles-from-headings (= 0.5.3)
+      jemoji (= 0.12.0)
+      kramdown (= 2.3.1)
+      kramdown-parser-gfm (= 1.1.0)
+      liquid (= 4.0.3)
+      mercenary (~> 0.3)
+      minima (= 2.5.1)
+      nokogiri (>= 1.10.4, < 2.0)
+      rouge (= 3.26.0)
+      terminal-table (~> 1.4)
+    github-pages-health-check (1.17.2)
+      addressable (~> 2.3)
+      dnsruby (~> 1.60)
+      octokit (~> 4.0)
+      public_suffix (>= 2.0.2, < 5.0)
+      typhoeus (~> 1.3)
+    html-pipeline (2.14.0)
+      activesupport (>= 2)
+      nokogiri (>= 1.4)
+    html-proofer (3.19.2)
+      addressable (~> 2.3)
+      mercenary (~> 0.3)
+      nokogumbo (~> 2.0)
+      parallel (~> 1.3)
+      rainbow (~> 3.0)
+      typhoeus (~> 1.3)
+      yell (~> 2.0)
+    http_parser.rb (0.6.0)
+    i18n (0.9.5)
+      concurrent-ruby (~> 1.0)
+    jekyll (3.9.0)
+      addressable (~> 2.4)
+      colorator (~> 1.0)
+      em-websocket (~> 0.5)
+      i18n (~> 0.7)
+      jekyll-sass-converter (~> 1.0)
+      jekyll-watch (~> 2.0)
+      kramdown (>= 1.17, < 3)
+      liquid (~> 4.0)
+      mercenary (~> 0.3.3)
+      pathutil (~> 0.9)
+      rouge (>= 1.7, < 4)
+      safe_yaml (~> 1.0)
+    jekyll-avatar (0.7.0)
+      jekyll (>= 3.0, < 5.0)
+    jekyll-coffeescript (1.1.1)
+      coffee-script (~> 2.2)
+      coffee-script-source (~> 1.11.1)
+    jekyll-commonmark (1.3.1)
+      commonmarker (~> 0.14)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-commonmark-ghpages (0.1.6)
+      commonmarker (~> 0.17.6)
+      jekyll-commonmark (~> 1.2)
+      rouge (>= 2.0, < 4.0)
+    jekyll-default-layout (0.1.4)
+      jekyll (~> 3.0)
+    jekyll-feed (0.15.1)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-gist (1.5.0)
+      octokit (~> 4.2)
+    jekyll-github-metadata (2.13.0)
+      jekyll (>= 3.4, < 5.0)
+      octokit (~> 4.0, != 4.4.0)
+    jekyll-mentions (1.6.0)
+      html-pipeline (~> 2.3)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-optional-front-matter (0.3.2)
+      jekyll (>= 3.0, < 5.0)
+    jekyll-paginate (1.1.0)
+    jekyll-readme-index (0.3.0)
+      jekyll (>= 3.0, < 5.0)
+    jekyll-redirect-from (0.16.0)
+      jekyll (>= 3.3, < 5.0)
+    jekyll-relative-links (0.6.1)
+      jekyll (>= 3.3, < 5.0)
+    jekyll-remote-theme (0.4.3)
+      addressable (~> 2.0)
+      jekyll (>= 3.5, < 5.0)
+      jekyll-sass-converter (>= 1.0, <= 3.0.0, != 2.0.0)
+      rubyzip (>= 1.3.0, < 3.0)
+    jekyll-sass-converter (1.5.2)
+      sass (~> 3.4)
+    jekyll-seo-tag (2.7.1)
+      jekyll (>= 3.8, < 5.0)
+    jekyll-sitemap (1.4.0)
+      jekyll (>= 3.7, < 5.0)
+    jekyll-swiss (1.0.0)
+    jekyll-theme-architect (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-cayman (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-dinky (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-hacker (0.1.2)
+      jekyll (> 3.5, < 5.0)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-leap-day (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-merlot (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-midnight (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-minimal (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-modernist (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-primer (0.5.4)
+      jekyll (> 3.5, < 5.0)
+      jekyll-github-metadata (~> 2.9)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-slate (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-tactile (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-theme-time-machine (0.1.1)
+      jekyll (~> 3.5)
+      jekyll-seo-tag (~> 2.0)
+    jekyll-titles-from-headings (0.5.3)
+      jekyll (>= 3.3, < 5.0)
+    jekyll-watch (2.2.1)
+      listen (~> 3.0)
+    jemoji (0.12.0)
+      gemoji (~> 3.0)
+      html-pipeline (~> 2.2)
+      jekyll (>= 3.0, < 5.0)
+    kramdown (2.3.1)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    liquid (4.0.3)
+    listen (3.5.1)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    mercenary (0.3.6)
+    minima (2.5.1)
+      jekyll (>= 3.5, < 5.0)
+      jekyll-feed (~> 0.9)
+      jekyll-seo-tag (~> 2.1)
+    minitest (5.14.4)
+    multipart-post (2.1.1)
+    nokogiri (1.11.7-x86_64-linux)
+      racc (~> 1.4)
+    nokogumbo (2.0.5)
+      nokogiri (~> 1.8, >= 1.8.4)
+    octokit (4.21.0)
+      faraday (>= 0.9)
+      sawyer (~> 0.8.0, >= 0.5.3)
+    parallel (1.21.0)
+    pathutil (0.16.2)
+      forwardable-extended (~> 2.6)
+    public_suffix (4.0.6)
+    racc (1.5.2)
+    rainbow (3.0.0)
+    rb-fsevent (0.11.0)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
+    rexml (3.2.5)
+    rouge (3.26.0)
+    ruby-enum (0.9.0)
+      i18n
+    ruby2_keywords (0.0.4)
+    rubyzip (2.3.0)
+    safe_yaml (1.0.5)
+    sass (3.7.4)
+      sass-listen (~> 4.0.0)
+    sass-listen (4.0.0)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+    sawyer (0.8.2)
+      addressable (>= 2.3.5)
+      faraday (> 0.8, < 2.0)
+    simpleidn (0.2.1)
+      unf (~> 0.1.4)
+    terminal-table (1.8.0)
+      unicode-display_width (~> 1.1, >= 1.1.1)
+    thread_safe (0.3.6)
+    typhoeus (1.4.0)
+      ethon (>= 0.9.0)
+    tzinfo (1.2.9)
+      thread_safe (~> 0.1)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    unicode-display_width (1.7.0)
+    yell (2.2.2)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  x86_64-linux
+  x86_64-linux-musl
+
+DEPENDENCIES
+  github-pages (~> 215)
+  html-proofer
+  minima (~> 2.5)
+  tzinfo (~> 1.2)
+  tzinfo-data
+  wdm (~> 0.1.1)
+
+BUNDLED WITH
+   2.2.21

docs/README.md

----
-title: Theodolite
-nav_order: 1
-permalink: /
----
+# Theodolite Docs
 
-# Theodolite
+Theodolite's docs are generated with Jekyll from Markdown files.
 
-> A theodolite is a precision optical instrument for measuring angles between designated visible points in the horizontal and vertical planes.  -- <cite>[Wikipedia](https://en.wikipedia.org/wiki/Theodolite)</cite>
+## Installation
 
-Theodolite is a framework for benchmarking the horizontal and vertical scalability of stream processing engines. It consists of three modules:
+To set up Jekyll run:
 
-## Theodolite Benchmarking Tool
+```sh
+gem install bundler
+bundle config set --local path 'vendor'
+bundle install
+```
 
-Theodolite aims to benchmark scalability of stream processing engines for real use cases. Microservices that apply stream processing techniques are usually deployed in elastic cloud environments. Hence, Theodolite's cloud-native benchmarking framework deploys its components in a cloud environment, orchestrated by Kubernetes. It is recommended to install Theodolite with the package manager Helm. The Theodolite Helm chart along with instructions how to install it can be found in the [`helm`](helm) directory.
+## Local Testing
 
-## Theodolite Analysis Tools
+For live serving the docs run:
 
-Theodolite's benchmarking method maps load intensities to the resource amounts that are required for processing them. A plot showing how resource demand evolves with an increasing load allows to draw conclusions about the scalability of a stream processing engine or its deployment. Theodolite provides Jupyter notebooks for creating such plots based on benchmarking results from the execution framework. More information can be found in [Theodolite analysis tool](analysis).
+```sh
+bundle exec jekyll serve
+```
 
-## Theodolite Benchmarks
+## Building
 
-Theodolite comes with 4 application benchmarks, which are based on typical use cases for stream processing within microservices. For each benchmark, a corresponding load generator is provided. Currently, this repository provides benchmark implementations for Apache Kafka Streams and Apache Flink. The benchmark sources can be found in [Thedolite benchmarks](theodolite-benchmarks).
+You can compile everything to HTML via:
 
-## How to Cite
+```sh
+bundle exec jekyll build
+```
 
-If you use Theodolite, please cite
+## CRD API Reference
 
-> Sören Henning and Wilhelm Hasselbring. (2021). Theodolite: Scalability Benchmarking of Distributed Stream Processing Engines in Microservice Architectures. Big Data Research, Volume 25. DOI: [10.1016/j.bdr.2021.100209](https://doi.org/10.1016/j.bdr.2021.100209). arXiv:[2009.00304](https://arxiv.org/abs/2009.00304).
+We use the [crdoc](https://github.com/fybrik/crdoc) tool to generate the API reference for our CRDs:
+
+```sh
+crdoc --resources ../theodolite/crd/ --template api-reference/crds.tmpl  --output api-reference/crds.md
+```
+
+With the following command, crdoc is installed and executed in Docker:
+
+```sh
+docker run --rm -v "`pwd`/../theodolite/crd/":/crd -v "`pwd`/api-reference":/api-reference golang sh -c "go install fybrik.io/crdoc@latest && crdoc --resources /crd/ --template /api-reference/crds.tmpl --output /api-reference/crds.md"
+```

docs/_config.yml

 title: "Theodolite"
+description: >-
+  Theodolite is a framework for benchmarking the horizontal and vertical
+  scalability of stream processing engines.
+
 remote_theme: pmarsceill/just-the-docs
 #color_scheme: "dark"
 aux_links:
     "Theodolite on GitHub":
       - "//github.com/cau-se/theodolite"
+
+exclude:
+  - "*.tmpl"
+  - Gemfile
+  - Gemfile.lock
+  - README.md
+  - vendor

docs/_sass/custom/custom.scss

+.theodolite-logo {
+    height: 18em;
+}
\ No newline at end of file

docs/crd-docu.md → docs/api-reference/crds.md

+---
+title: Theodolite CRDs
+has_children: false
+parent: API Reference
+nav_order: 1
+---
+
+
 # API Reference
 
 Packages:
@@ -52,45 +60,18 @@ Resource Types:
       <td>Refer to the Kubernetes API documentation for the fields of the `metadata` field.</td>
       <td>true</td>
       </tr><tr>
-        <td><b><a href="#benchmarkstatus">status</a></b></td>
+        <td><b><a href="#benchmarkspec">spec</a></b></td>
         <td>object</td>
         <td>
           <br/>
         </td>
-        <td>false</td>
+        <td>true</td>
       </tr><tr>
-        <td><b><a href="#benchmarkspec">spec</a></b></td>
+        <td><b><a href="#benchmarkstatus">status</a></b></td>
         <td>object</td>
         <td>
           <br/>
         </td>
-        <td>true</td>
-      </tr></tbody>
-</table>
-
-
-### benchmark.status
-<sup><sup>[↩ Parent](#benchmark)</sup></sup>
-
-
-
-
-
-<table>
-    <thead>
-        <tr>
-            <th>Name</th>
-            <th>Type</th>
-            <th>Description</th>
-            <th>Required</th>
-        </tr>
-    </thead>
-    <tbody><tr>
-        <td><b>resourceSetsState</b></td>
-        <td>string</td>
-        <td>
-          The status of a Benchmark indicates whether all resources are available to start the benchmark or not.<br/>
-        </td>
         <td>false</td>
       </tr></tbody>
 </table>
@@ -113,24 +94,6 @@ Resource Types:
         </tr>
     </thead>
     <tbody><tr>
-        <td><b><a href="#benchmarkspecinfrastructure">infrastructure</a></b></td>
-        <td>object</td>
-        <td>
-          (Optional) A list of file names that reference Kubernetes resources that are deployed on the cluster to create the required infrastructure.<br/>
-          <br/>
-            <i>Default</i>: map[]<br/>
-        </td>
-        <td>false</td>
-      </tr><tr>
-        <td><b>name</b></td>
-        <td>string</td>
-        <td>
-          This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/>
-          <br/>
-            <i>Default</i>: <br/>
-        </td>
-        <td>false</td>
-      </tr><tr>
         <td><b><a href="#benchmarkspeckafkaconfig">kafkaConfig</a></b></td>
         <td>object</td>
         <td>
@@ -165,135 +128,22 @@ Resource Types:
           The appResourceSets specifies all Kubernetes resources required to start the sut. A resourceSet can be either a configMap resourceSet or a fileSystem resourceSet.<br/>
         </td>
         <td>true</td>
-      </tr></tbody>
-</table>
-
-
-### benchmark.spec.infrastructure
-<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup>
-
-
-
-(Optional) A list of file names that reference Kubernetes resources that are deployed on the cluster to create the required infrastructure.
-
-<table>
-    <thead>
-        <tr>
-            <th>Name</th>
-            <th>Type</th>
-            <th>Description</th>
-            <th>Required</th>
-        </tr>
-    </thead>
-    <tbody><tr>
-        <td><b><a href="#benchmarkspecinfrastructureresourcesindex">resources</a></b></td>
-        <td>[]object</td>
-        <td>
-          <br/>
-          <br/>
-            <i>Default</i>: []<br/>
-        </td>
-        <td>false</td>
-      </tr></tbody>
-</table>
-
-
-### benchmark.spec.infrastructure.resources[index]
-<sup><sup>[↩ Parent](#benchmarkspecinfrastructure)</sup></sup>
-
-
-
-
-
-<table>
-    <thead>
-        <tr>
-            <th>Name</th>
-            <th>Type</th>
-            <th>Description</th>
-            <th>Required</th>
-        </tr>
-    </thead>
-    <tbody><tr>
-        <td><b><a href="#benchmarkspecinfrastructureresourcesindexconfigmap">configMap</a></b></td>
-        <td>object</td>
-        <td>
-          The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap.<br/>
-        </td>
-        <td>false</td>
       </tr><tr>
-        <td><b><a href="#benchmarkspecinfrastructureresourcesindexfilesystem">fileSystem</a></b></td>
+        <td><b><a href="#benchmarkspecinfrastructure">infrastructure</a></b></td>
         <td>object</td>
         <td>
-          The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.<br/>
-        </td>
-        <td>false</td>
-      </tr></tbody>
-</table>
-
-
-### benchmark.spec.infrastructure.resources[index].configMap
-<sup><sup>[↩ Parent](#benchmarkspecinfrastructureresourcesindex)</sup></sup>
-
-
-
-The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap.
-
-<table>
-    <thead>
-        <tr>
-            <th>Name</th>
-            <th>Type</th>
-            <th>Description</th>
-            <th>Required</th>
-        </tr>
-    </thead>
-    <tbody><tr>
-        <td><b>files</b></td>
-        <td>[]string</td>
-        <td>
-          (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded.<br/>
+          (Optional) A list of file names that reference Kubernetes resources that are deployed on the cluster to create the required infrastructure.<br/>
+          <br/>
+            <i>Default</i>: map[]<br/>
         </td>
         <td>false</td>
       </tr><tr>
         <td><b>name</b></td>
         <td>string</td>
         <td>
-          The name of the configMap<br/>
-        </td>
-        <td>false</td>
-      </tr></tbody>
-</table>
-
-
-### benchmark.spec.infrastructure.resources[index].fileSystem
-<sup><sup>[↩ Parent](#benchmarkspecinfrastructureresourcesindex)</sup></sup>
-
-
-
-The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
-
-<table>
-    <thead>
-        <tr>
-            <th>Name</th>
-            <th>Type</th>
-            <th>Description</th>
-            <th>Required</th>
-        </tr>
-    </thead>
-    <tbody><tr>
-        <td><b>files</b></td>
-        <td>[]string</td>
-        <td>
-          (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded.<br/>
-        </td>
-        <td>false</td>
-      </tr><tr>
-        <td><b>path</b></td>
-        <td>string</td>
-        <td>
-          The path to the folder which contains the Kubernetes manifests files.<br/>
+          This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/>
+          <br/>
+            <i>Default</i>: <br/>
         </td>
         <td>false</td>
       </tr></tbody>
@@ -351,6 +201,15 @@ Contains the Kafka configuration.
         </tr>
     </thead>
     <tbody><tr>
+        <td><b>name</b></td>
+        <td>string</td>
+        <td>
+          The name of the topic.<br/>
+          <br/>
+            <i>Default</i>: <br/>
+        </td>
+        <td>true</td>
+      </tr><tr>
         <td><b>numPartitions</b></td>
         <td>integer</td>
         <td>
@@ -377,15 +236,6 @@ Contains the Kafka configuration.
             <i>Default</i>: 0<br/>
         </td>
         <td>false</td>
-      </tr><tr>
-        <td><b>name</b></td>
-        <td>string</td>
-        <td>
-          The name of the topic.<br/>
-          <br/>
-            <i>Default</i>: <br/>
-        </td>
-        <td>true</td>
       </tr></tbody>
 </table>
 
@@ -572,15 +422,6 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
         </tr>
     </thead>
     <tbody><tr>
-        <td><b>properties</b></td>
-        <td>map[string]string</td>
-        <td>
-          (Optional) Patcher specific additional arguments.<br/>
-          <br/>
-            <i>Default</i>: map[]<br/>
-        </td>
-        <td>false</td>
-      </tr><tr>
         <td><b>resource</b></td>
         <td>string</td>
         <td>
@@ -598,6 +439,15 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
             <i>Default</i>: <br/>
         </td>
         <td>true</td>
+      </tr><tr>
+        <td><b>properties</b></td>
+        <td>map[string]string</td>
+        <td>
+          (Optional) Patcher specific additional arguments.<br/>
+          <br/>
+            <i>Default</i>: map[]<br/>
+        </td>
+        <td>false</td>
       </tr></tbody>
 </table>
 
@@ -653,15 +503,6 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
         </tr>
     </thead>
     <tbody><tr>
-        <td><b>properties</b></td>
-        <td>map[string]string</td>
-        <td>
-          (Optional) Patcher specific additional arguments.<br/>
-          <br/>
-            <i>Default</i>: map[]<br/>
-        </td>
-        <td>false</td>
-      </tr><tr>
         <td><b>resource</b></td>
         <td>string</td>
         <td>
@@ -679,6 +520,15 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
             <i>Default</i>: <br/>
         </td>
         <td>true</td>
+      </tr><tr>
+        <td><b>properties</b></td>
+        <td>map[string]string</td>
+        <td>
+          (Optional) Patcher specific additional arguments.<br/>
+          <br/>
+            <i>Default</i>: map[]<br/>
+        </td>
+        <td>false</td>
       </tr></tbody>
 </table>
 
@@ -813,15 +663,13 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
       </tr></tbody>
 </table>
 
-## execution
-<sup><sup>[↩ Parent](#theodolitecomv1 )</sup></sup>
-
-
-
 
+### benchmark.spec.infrastructure
+<sup><sup>[↩ Parent](#benchmarkspec)</sup></sup>
 
 
 
+(Optional) A list of file names that reference Kubernetes resources that are deployed on the cluster to create the required infrastructure.
 
 <table>
     <thead>
@@ -833,46 +681,58 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
         </tr>
     </thead>
     <tbody><tr>
-      <td><b>apiVersion</b></td>
-      <td>string</td>
-      <td>theodolite.com/v1</td>
-      <td>true</td>
-      </tr>
-      <tr>
-      <td><b>kind</b></td>
-      <td>string</td>
-      <td>execution</td>
-      <td>true</td>
-      </tr>
-      <tr>
-      <td><b><a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#objectmeta-v1-meta">metadata</a></b></td>
-      <td>object</td>
-      <td>Refer to the Kubernetes API documentation for the fields of the `metadata` field.</td>
-      <td>true</td>
-      </tr><tr>
-        <td><b><a href="#executionstatus">status</a></b></td>
-        <td>object</td>
+        <td><b><a href="#benchmarkspecinfrastructureresourcesindex">resources</a></b></td>
+        <td>[]object</td>
         <td>
           <br/>
+          <br/>
+            <i>Default</i>: []<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
+### benchmark.spec.infrastructure.resources[index]
+<sup><sup>[↩ Parent](#benchmarkspecinfrastructure)</sup></sup>
+
+
+
+
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b><a href="#benchmarkspecinfrastructureresourcesindexconfigmap">configMap</a></b></td>
+        <td>object</td>
+        <td>
+          The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap.<br/>
         </td>
         <td>false</td>
       </tr><tr>
-        <td><b><a href="#executionspec">spec</a></b></td>
+        <td><b><a href="#benchmarkspecinfrastructureresourcesindexfilesystem">fileSystem</a></b></td>
         <td>object</td>
         <td>
-          <br/>
+          The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.<br/>
         </td>
-        <td>true</td>
+        <td>false</td>
       </tr></tbody>
 </table>
 
 
-### execution.status
-<sup><sup>[↩ Parent](#execution)</sup></sup>
-
+### benchmark.spec.infrastructure.resources[index].configMap
+<sup><sup>[↩ Parent](#benchmarkspecinfrastructureresourcesindex)</sup></sup>
 
 
 
+The configMap resourceSet loads the Kubernetes manifests from an Kubernetes configMap.
 
 <table>
     <thead>
@@ -884,25 +744,59 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
         </tr>
     </thead>
     <tbody><tr>
-        <td><b>executionDuration</b></td>
+        <td><b>files</b></td>
+        <td>[]string</td>
+        <td>
+          (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded.<br/>
+        </td>
+        <td>false</td>
+      </tr><tr>
+        <td><b>name</b></td>
         <td>string</td>
         <td>
-          Duration of the execution in seconds<br/>
+          The name of the configMap<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
+### benchmark.spec.infrastructure.resources[index].fileSystem
+<sup><sup>[↩ Parent](#benchmarkspecinfrastructureresourcesindex)</sup></sup>
+
+
+
+The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b>files</b></td>
+        <td>[]string</td>
+        <td>
+          (Optional) Specifies which files from the configMap should be loaded. If this field is not set, all files are loaded.<br/>
         </td>
         <td>false</td>
       </tr><tr>
-        <td><b>executionState</b></td>
+        <td><b>path</b></td>
         <td>string</td>
         <td>
-          <br/>
+          The path to the folder which contains the Kubernetes manifests files.<br/>
         </td>
         <td>false</td>
       </tr></tbody>
 </table>
 
 
-### execution.spec
-<sup><sup>[↩ Parent](#execution)</sup></sup>
+### benchmark.status
+<sup><sup>[↩ Parent](#benchmark)</sup></sup>
 
 
 
@@ -918,15 +812,86 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
         </tr>
     </thead>
     <tbody><tr>
-        <td><b>name</b></td>
+        <td><b>resourceSetsState</b></td>
         <td>string</td>
         <td>
-          This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/>
-          <br/>
-            <i>Default</i>: <br/>
+          The status of a Benchmark indicates whether all resources are available to start the benchmark or not.<br/>
         </td>
         <td>false</td>
+      </tr></tbody>
+</table>
+
+## execution
+<sup><sup>[↩ Parent](#theodolitecomv1 )</sup></sup>
+
+
+
+
+
+
+
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+      <td><b>apiVersion</b></td>
+      <td>string</td>
+      <td>theodolite.com/v1</td>
+      <td>true</td>
+      </tr>
+      <tr>
+      <td><b>kind</b></td>
+      <td>string</td>
+      <td>execution</td>
+      <td>true</td>
+      </tr>
+      <tr>
+      <td><b><a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#objectmeta-v1-meta">metadata</a></b></td>
+      <td>object</td>
+      <td>Refer to the Kubernetes API documentation for the fields of the `metadata` field.</td>
+      <td>true</td>
       </tr><tr>
+        <td><b><a href="#executionspec">spec</a></b></td>
+        <td>object</td>
+        <td>
+          <br/>
+        </td>
+        <td>true</td>
+      </tr><tr>
+        <td><b><a href="#executionstatus">status</a></b></td>
+        <td>object</td>
+        <td>
+          <br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
+### execution.spec
+<sup><sup>[↩ Parent](#execution)</sup></sup>
+
+
+
+
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
         <td><b>benchmark</b></td>
         <td>string</td>
         <td>
@@ -968,6 +933,15 @@ The fileSystem resourceSet loads the Kubernetes manifests from the filesystem.
           List of resource values for the specified resource type.<br/>
         </td>
         <td>true</td>
+      </tr><tr>
+        <td><b>name</b></td>
+        <td>string</td>
+        <td>
+          This field exists only for technical reasons and should not be set by the user. The value of the field will be overwritten.<br/>
+          <br/>
+            <i>Default</i>: <br/>
+        </td>
+        <td>false</td>
       </tr></tbody>
 </table>
 
@@ -1023,15 +997,6 @@ Patcher used to patch a resource
         </tr>
     </thead>
     <tbody><tr>
-        <td><b>properties</b></td>
-        <td>map[string]string</td>
-        <td>
-          (Optional) Patcher specific additional arguments.<br/>
-          <br/>
-            <i>Default</i>: map[]<br/>
-        </td>
-        <td>false</td>
-      </tr><tr>
         <td><b>resource</b></td>
         <td>string</td>
         <td>
@@ -1049,6 +1014,15 @@ Patcher used to patch a resource
             <i>Default</i>: <br/>
         </td>
         <td>true</td>
+      </tr><tr>
+        <td><b>properties</b></td>
+        <td>map[string]string</td>
+        <td>
+          (Optional) Patcher specific additional arguments.<br/>
+          <br/>
+            <i>Default</i>: map[]<br/>
+        </td>
+        <td>false</td>
       </tr></tbody>
 </table>
 
@@ -1070,13 +1044,6 @@ Defines the overall parameter for the execution.
         </tr>
     </thead>
     <tbody><tr>
-        <td><b>loadGenerationDelay</b></td>
-        <td>integer</td>
-        <td>
-          Seconds to wait between the start of the SUT and the load generator.<br/>
-        </td>
-        <td>false</td>
-      </tr><tr>
         <td><b>duration</b></td>
         <td>integer</td>
         <td>
@@ -1104,6 +1071,13 @@ Defines the overall parameter for the execution.
           Defines the used strategy for the execution, either 'LinearSearch' or 'BinarySearch'<br/>
         </td>
         <td>true</td>
+      </tr><tr>
+        <td><b>loadGenerationDelay</b></td>
+        <td>integer</td>
+        <td>
+          Seconds to wait between the start of the SUT and the load generator.<br/>
+        </td>
+        <td>false</td>
       </tr></tbody>
 </table>
 
@@ -1193,15 +1167,6 @@ Specifies the scaling resource that is benchmarked.
         </tr>
     </thead>
     <tbody><tr>
-        <td><b>properties</b></td>
-        <td>map[string]string</td>
-        <td>
-          (Optional) SLO specific additional arguments.<br/>
-          <br/>
-            <i>Default</i>: map[]<br/>
-        </td>
-        <td>false</td>
-      </tr><tr>
         <td><b>offset</b></td>
         <td>integer</td>
         <td>
@@ -1222,5 +1187,48 @@ Specifies the scaling resource that is benchmarked.
           The type of the SLO. It must match 'lag trend'.<br/>
         </td>
         <td>true</td>
+      </tr><tr>
+        <td><b>properties</b></td>
+        <td>map[string]string</td>
+        <td>
+          (Optional) SLO specific additional arguments.<br/>
+          <br/>
+            <i>Default</i>: map[]<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
+### execution.status
+<sup><sup>[↩ Parent](#execution)</sup></sup>
+
+
+
+
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b>executionDuration</b></td>
+        <td>string</td>
+        <td>
+          Duration of the execution in seconds<br/>
+        </td>
+        <td>false</td>
+      </tr><tr>
+        <td><b>executionState</b></td>
+        <td>string</td>
+        <td>
+          <br/>
+        </td>
+        <td>false</td>
       </tr></tbody>
 </table>
\ No newline at end of file

docs/api-reference/crds.tmpl

+---
+title: Theodolite CRDs
+has_children: false
+parent: API Reference
+nav_order: 1
+---
+
+
+# API Reference
+
+Packages:
+{{range .Groups}}
+- [{{.Group}}/{{.Version}}](#{{ anchorize (printf "%s/%s" .Group .Version) }})
+{{- end -}}{{/* range .Groups */}}
+
+{{- range .Groups }}
+{{- $group := . }}
+
+# {{.Group}}/{{.Version}}
+
+Resource Types:
+{{range .Kinds}}
+- [{{.Name}}](#{{ anchorize .Name }})
+{{end}}{{/* range .Kinds */}}
+
+{{range .Kinds}}
+{{$kind := .}}
+## {{.Name}}
+<sup><sup>[↩ Parent](#{{ anchorize (printf "%s/%s" $group.Group $group.Version) }} )</sup></sup>
+
+{{range .Types}}
+
+{{if not .IsTopLevel}}
+### {{.Name}}
+{{if .ParentKey}}<sup><sup>[↩ Parent](#{{.ParentKey}})</sup></sup>{{end}}
+{{end}}
+
+
+{{.Description}}
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody>
+      {{- if .IsTopLevel -}}
+      <tr>
+      <td><b>apiVersion</b></td>
+      <td>string</td>
+      <td>{{$group.Group}}/{{$group.Version}}</td>
+      <td>true</td>
+      </tr>
+      <tr>
+      <td><b>kind</b></td>
+      <td>string</td>
+      <td>{{$kind.Name}}</td>
+      <td>true</td>
+      </tr>
+      <tr>
+      <td><b><a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#objectmeta-v1-meta">metadata</a></b></td>
+      <td>object</td>
+      <td>Refer to the Kubernetes API documentation for the fields of the `metadata` field.</td>
+      <td>true</td>
+      </tr>
+      {{- end -}}
+      {{- range .Fields -}}
+      <tr>
+        <td><b>{{if .TypeKey}}<a href="#{{.TypeKey}}">{{.Name}}</a>{{else}}{{.Name}}{{end}}</b></td>
+        <td>{{.Type}}</td>
+        <td>
+          {{.Description}}<br/>
+          {{- if or .Schema.Format .Schema.Enum .Schema.Default .Schema.Minimum .Schema.Maximum }}
+          <br/>
+          {{- end}}
+          {{- if .Schema.Format }}
+            <i>Format</i>: {{ .Schema.Format }}<br/>
+          {{- end }}
+          {{- if .Schema.Enum }}
+            <i>Enum</i>: {{ .Schema.Enum | toStrings | join ", " }}<br/>
+          {{- end }}
+          {{- if .Schema.Default }}
+            <i>Default</i>: {{ .Schema.Default }}<br/>
+          {{- end }}
+          {{- if .Schema.Minimum }}
+            <i>Minimum</i>: {{ .Schema.Minimum }}<br/>
+          {{- end }}
+          {{- if .Schema.Maximum }}
+            <i>Maximum</i>: {{ .Schema.Maximum }}<br/>
+          {{- end }}
+        </td>
+        <td>{{.Required}}</td>
+      </tr>
+      {{- end -}}
+    </tbody>
+</table>
+
+{{- end}}{{/* range .Types */}}
+{{- end}}{{/* range .Kinds */}}
+{{- end}}{{/* range .Groups */}}
\ No newline at end of file

docs/api-reference/index.md

+---
+title: API Reference
+has_children: true
+nav_order: 9
+---
\ No newline at end of file

docs/patchers.md → docs/api-reference/patchers.md

-## Patchers
+---
+title: Patchers
+has_children: false
+parent: API Reference
+nav_order: 2
+---
 
-* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
+# Patchers
+
+Patchers can be seen as functions which take a value as input and modify a Kubernetes resource in a patcher-specific way.
+
+## Available Patchers
+
+* **ReplicaPatcher**: Modifies the number of replicas for a Kubernetes deployment.
   * **type**: "ReplicaPatcher"
   * **resource**: "uc1-kstreams-deployment.yaml"
 
-* **NumSensorsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (value + 15_000 - 1) / 15_000
+* **NumSensorsLoadGeneratorReplicaPatcher**: Modifies the number of load generators, according to the following formula: *(value + loadGenMaxRecords - 1) / loadGenMaxRecords*
   * **type**: "NumSensorsLoadGeneratorReplicaPatcher"
   * **resource**: "uc1-load-generator-deployment.yaml"
+  * **properties**:
+    * loadGenMaxRecords: 150000
 
-* **NumNestedGroupsLoadGeneratorReplicaPatcher**: Allows to scale the nummer of load generators. Scales arcording to the following formular: (4^(value) + 15_000 -1) /15_000
+* **NumNestedGroupsLoadGeneratorReplicaPatcher**: Modifies the number of load generators, according to the following formula: *(4^(value) + loadGenMaxRecords - 1) / loadGenMaxRecords*
   * **type**: "NumNestedGroupsLoadGeneratorReplicaPatcher"
   * **resource**: "uc1-load-generator-deployment.yaml"
+  * **properties**:
+    * loadGenMaxRecords: 150000
 
-* **ReplicaPatcher**: Allows to modify the number of Replicas for a kubernetes deployment.
-  * **type**: "ReplicaPatcher"
-  * **resource**: "uc1-kstreams-deployment.yaml"
-
-* **EnvVarPatcher**: Allows to modify the value of an environment variable for a container in a kubernetes deployment. 
+* **EnvVarPatcher**: Modifies the value of an environment variable for a container in a Kubernetes deployment. 
   * **type**: "EnvVarPatcher"
   * **resource**: "uc1-load-generator-deployment.yaml"
   * **properties**:
     * container: "workload-generator"
     * variableName: "NUM_SENSORS"
 
-* **NodeSelectorPatcher**: Changes the node selection field in kubernetes resources.
+* **NodeSelectorPatcher**: Changes the node selection field in Kubernetes resources.
   * **type**: "NodeSelectorPatcher"
   * **resource**: "uc1-load-generator-deployment.yaml"
   * **properties**:
     * variableName: "env"
-  * **value**: "prod"
+  * **example value**: "prod"
 
-* **ResourceLimitPatcher**: Changes the resource limit for a kubernetes resource.
+* **ResourceLimitPatcher**: Changes the resource limit for a Kubernetes resource.
   * **resource**: "uc1-kstreams-deployment.yaml"
   * **properties**:
     * container: "uc-application"
     * variableName: "cpu" or "memory"
-  * **value**:"1000m" or "2Gi"
+  * **example value**:"1000m" or "2Gi"
   
-* **SchedulerNamePatcher**: Changes the sheduler for kubernetes resources.
+* **SchedulerNamePatcher**: Changes the scheduler for Kubernetes resources.
   * **type**: "SchedulerNamePatcher"
   * **resource**: "uc1-kstreams-deployment.yaml"
-  * **value**: "random-scheduler"
+  * **example value**: "random-scheduler"
 
-* **ImagePatcher**: Changes the image of a kubernetes resource. Currently not fully implemented.
+* **ImagePatcher**: Changes the image of a Kubernetes resource. **Currently not fully implemented.**
   * **type**: "ImagePatcher"
   * **resource**: "uc1-kstreams-deployment.yaml"
   * **properties**:
     * container: "uc-application"
-  * **value**: "dockerhubrepo/imagename"
+  * **example value**: "dockerhub-org/image-name"

docs/benchmarks-and-executions.md

+---
+title: Fundamental Concepts
+has_children: false
+nav_order: 2
+---
+
+# Benchmarks and Executions
+
+In Theodolite, we distinguish between the static description of a scalability benchmark and its execution.
+
+## Benchmarks
+
+Benchmarks define what should be executed in scalability experiments. They
+consist of a system under test (SUT) and an associated load generator, where
+both SUT and load generator are represented as sets of Kubernetes resources
+such as Pods, Services, or ConfigMaps.
+Additionally, benchmarks define one or more load and resource types, scalability
+can be evaluated for.
+
+[TODO]: # (Link to metrics)
+
+Benchmarks are portable, that is they can be provided by standardization
+organizations, researchers, etc. and installed by other researchers,
+engineers, etc.
+They do not have a life-cycle. Instead, to run a benchmark, an execution of a
+benchmark is created and passed to Theodolite.
+
+## Execution
+
+An execution represents a one-time execution of a benchmark with a
+specific configuration. Hence, an execution refers to a benchmark and one of
+the load and resource types defined by that benchmark. Additionally, executions
+allow to alter the SUT or the load generator to evaluate regarding specific
+configurations.
+
+Executions also describe details regarding the scalability measurement method,
+such as for long experiment should be performed or how often they should be
+repeated.
+
+In contrast to benchmarks, execution have a life-cycle. They can be planned,
+executed, or aborted. Each execution of a benchmark is represented by an
+individual entity. This supports repeatability as executions can be archived
+and shared.

docs/creating-a-benchmark.md

+---
+title: Creating Benchmarks
+has_children: false
+nav_order: 5
+---
+
+# Creating a Benchmark
+
+Please note that to simply run a benchmark, it is not required to define one. Theodolite comes with a [set of benchmarks](theodolite-benchmarks), which are ready to be executed. See the [fundamental concepts](benchmarks-and-executions) page to learn more about our distinction between benchmarks and executions.
+
+A typical benchmark looks like this:
+
+```yaml
+apiVersion: theodolite.com/v1
+kind: benchmark
+metadata:
+  name: example-benchmark
+spec:
+  sut:
+    resources:
+      - configMap:
+         name: "example-configmap"
+         files:
+           - "uc1-kstreams-deployment.yaml"
+  loadGenerator:
+    resources:
+      - configMap:
+         name: "example-configmap"
+         files:
+            - uc1-load-generator-service.yaml
+            - uc1-load-generator-deployment.yaml
+  loadTypes:
+    - typeName: "NumSensors"
+      patchers:
+        - type: "EnvVarPatcher"
+          resource: "uc1-load-generator-deployment.yaml"
+          properties:
+            variableName: "NUM_SENSORS"
+            container: "workload-generator"
+        - type: "NumSensorsLoadGeneratorReplicaPatcher"
+          resource: "uc1-load-generator-deployment.yaml"
+          properties:
+            loadGenMaxRecords: "150000"
+  kafkaConfig:
+    bootstrapServer: "theodolite-cp-kafka:9092"
+    topics:
+      - name: "input"
+        numPartitions: 40
+        replicationFactor: 1
+      - name: "theodolite-.*"
+        removeOnly: True
+
+```
+
+## System under Test (SUT), Load Generator and Infrastructure
+
+In Thedolite, the system under test (SUT), the load generator as well as additional infrastructure (e.g., a middleware) are described by Kubernetes resources files.
+All resources defined for the SUT and the load generator are started and stopped for each SLO experiment, with SUT resources being started before the load generator.
+Infrastructure resources live over the entire duration of a benchmark run. They avoid time-consuming recreation of software components like middlewares, but should be used with caution to not let previous SLO experiments influence latte ones.
+
+### Resources
+
+#### ConfigMap
+
+The recommended way to link Kubernetes resources files from a Benchmark is by bundling them in one or multiple ConfigMaps and refer to that ConfigMap from `sut.resources`, `loadGenerator.resources` or `infrastructure.resources`.
+To create a ConfigMap from all the Kubernetes resources in a directory run:
+
+```sh
+kubectl create configmap <configmap-name> --from-file=<path-to-resource-dir>
+```
+
+Add an item such as the following one to the `resources` list of the `sut`, `loadGenerator` or `infrastructure` fields.
+
+```yaml
+configMap:
+  name: example-configmap
+  files:
+  - example-deployment.yaml
+  - example-service.yaml
+```
+
+#### Filesystem
+
+Alternatively, resources can also be read from the filesystem, Theodolite has access to. This usually requires that the Benchmark resources are available in a volume, which is mounted into the Theodolite container.
+
+```yaml
+filesystem:
+  path: example/path/to/files
+  files:
+  - example-deployment.yaml
+  - example-service.yaml
+```
+
+<!-- ### Before and after actions -->
+
+
+
+
+<!--
+A Benchmark refers to other Kubernetes resources (e.g., Deployments, Services, ConfigMaps), which describe the system under test, the load generator and infrastructure components such as a middleware used in the benchmark. To manage those resources, Theodolite needs to have access to them. This is done by bundling resources in ConfigMaps.
+
+Suppose the resources needed by your benchmark are defined as YAML files, located in the `resources` directory. You can put them into the ConfigMap `benchmark-resources-custom` by running:
+-->
+
+
+## Load and Resource Types
+
+Benchmarks need to specify at least one supported load and resource type for which scalability can be benchmarked.
+
+Load and resource types are described by a name (used for reference from an Execution) and a list of patchers.
+If a benchmark is [executed by an Execution](running-benchmarks), these patchers are used to configure SUT and load generator according to the [load and resource values](creating-an-execution) set in the Execution.
+
+## Kafka Configuration
+
+Theodolite allows to automatically create and remove Kafka topics for each SLO experiment.
+Use the `removeOnly: True` property for topics which are created automatically by the SUT.
+For those topics, also wildcards are allowed in the topic name.
+
+
+<!-- Further information: API Reference -->
+<!-- Further information: How to deploy -->

docs/creating-an-execution.md

+---
+title: Creating an Execution
+has_children: false
+parent: Running Benchmarks
+nav_order: 6
+---
+
+# Creating an Execution
+
+Theodolite Executions look similar to the following example.
+
+<!-- TODO align with upstream -->
+```yaml
+apiVersion: theodolite.com/v1
+kind: execution
+metadata:
+  name: theodolite-example-execution
+spec:
+  benchmark: "uc1-kstreams"
+  load:
+    loadType: "NumSensors"
+    loadValues: [25000, 50000]
+  resources:
+    resourceType: "Instances"
+    resourceValues: [1, 2]
+  slos:
+    - sloType: "lag trend"
+      prometheusUrl: "http://prometheus-operated:9090"
+      offset: 0
+      properties:
+        threshold: 2000
+        externalSloUrl: "http://localhost:80/evaluate-slope"
+        warmup: 60 # in seconds
+  execution:
+    strategy: "LinearSearch"
+    duration: 300 # in seconds
+    repetitions: 1
+    loadGenerationDelay: 30 # in seconds
+    restrictions:
+      - "LowerBound"
+  configOverrides:
+    - patcher:
+        type: "SchedulerNamePatcher"
+        resource: "uc1-kstreams-deployment.yaml"
+      value: "random-scheduler"
+```
+
+## Naming and Labeling
+
+Similar to [Kubernetes Jobs](https://kubernetes.io/docs/concepts/workloads/controllers/job/), Theodolite Executions are not automatically deleted after they are finished. Therefore, it is recommended to choose a meaningful name for your Execution. Additionally, you can add labels to your Executions as with any other Kubernetes resource.
+
+## Reference to a Benchmark
+
+An Execution always refers to a Benchmark. For the Execution to run, the Benchmark must be registered with Kubernetes and it must be in state *Ready*. If this is not the case, the Execution will remain in state *Pending*.
+
+As a Benchmark may define multiple supported load and resource types, an Execution has to pick exactly one of each by its name. Additionally, it defines the set of load values and resource values the benchmark should be executed with. Both these values are represented as integers, which are interpreted in a [Benchmark-specific way](creating-a-benchmark) to configure the SUT and load generator.
+
+## Definition of SLOs
+
+SLOs provide a way to quantify whether a certain load intensity can be handled by a certain amount of provisioned resources.
+An Execution must at least specify one SLO to be checked.
+
+## Experimental Setup
+
+According to Theodolite's measurement method, isolated SLO experiments are performed for different combinations of load intensity and resource amounts.
+The experimental setup can be configured by:
+
+* A search strategy (`strategy`), which determines which load and resource combinations should be tested. Supported values are `FullSearch`, `LinearSearch` and `BinarySearch`. Additionally, a `restrictions` can be set to `LowerBound`.
+* The `duration` per SLO experiment in seconds.
+* The number of repetitions (`repetitions`) for each SLO experiment.
+* A `loadGenerationDelay`, specifying the time in seconds before the load generation starts.
+
+## Configuration Overrides
+
+In cases where only small modifications of a system under test should be benchmarked, it is not necessarily required to [create a new benchmark](creating-a-benchmark).
+Instead, also Executions allow to do small reconfigurations, such as switching on or off a specific Pod scheduler.
+
+This is done by defining `configOverrides` in the Execution. Each override consists of a patcher, defining which Kubernetes resource should be patched in which way, and a value the patcher is applied with.
+
+<!-- Further information: API Reference -->
+<!-- Further information: How to run -->

docs/development/index.md

+---
+title: Development
+has_children: true
+nav_order: 8
+---
\ No newline at end of file

docs/release-process.md → docs/development/release-process.md

 ---
 title: Release Process
 has_children: false
-nav_order: 2
+parent: Development
+nav_order: 1
 ---
 
 # Release Process
@@ -26,7 +27,7 @@ again be merged into master.
 
     2. Update both, the `version` and the `appVersion` fields, in the Helm `Charts.yaml` file to `0.3.1`.
 
-    3. Update `codemeta.json` to match the new version. In particular, make sure that `version` points to the version you are releasing and `dateModified` points to the date you are relasing this version. [CodeMeata generator](https://codemeta.github.io/codemeta-generator/) may help you in updating the file.
+    3. Update `codemeta.json` to match the new version. In particular, make sure that `version` points to the version you are releasing and `dateModified` points to the date you are relasing this version. [CodeMeta generator](https://codemeta.github.io/codemeta-generator/) may help you in updating the file.
 
     4. Update `CITATION.cff` to match the new version. At least update the `version` field.
 
@@ -36,9 +37,9 @@ again be merged into master.
 
 6. Commit these changes to the `v0.3` branch.
 
-7. Tag this commit `v0.3.1` (can be done via GitLab). The corresponding Docker images will be uploaded.
+7. Tag this commit `v0.3.1` (can be done via GitLab). The corresponding Docker images will be uploaded from the CI/CD pipeline.
 
-8. Create *releases* on GitLab and GitHub. Upload the generated Helm package to these releases.
+8. Create *releases* on GitLab and GitHub. Upload the generated Helm package to these releases via the UIs of GitLab and GitHub.
 
 9. Switch to the `master` branch.
 

docs/index.md

+---
+title: Home
+nav_order: 1
+#nav_exclude: true
+permalink: /
+---
+
+![Theodolite](assets/theodolite-stacked-transparent.svg){: .d-block .mx-auto .mb-8 .theodolite-logo }
+
+
+Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications.
+{: .fs-6 .fw-300 .text-center }
+
+[Get started now](quickstart){: .btn .btn-primary .fs-5 .mb-4 .mb-md-0 .mr-4 }
+[Learn the underlying concepts](benchmarks-and-executions){: .btn .fs-5 .mb-4 .mb-md-0 }
+{: .text-center }
+
+---

docs/installation.md

+---
+title: Installation
+has_children: false
+nav_order: 3
+---
+
+# Installing Theodolite
+
+The easiest option to install Theodolite is using [Helm](https://helm.sh).
+To install Theodolite with all its dependencies run:
+
+```sh
+helm repo add theodolite https://cau-se.github.io/theodolite
+helm repo update
+helm install theodolite theodolite/theodolite
+```
+
+This installs Theodolite in operator mode. Operator mode is the easiest to use, but requires some permissions in the installation. If those cannot be granted, Theodolite can also be installed for standalone mode. 
+
+
+## Installation Options
+
+As usual, the installation via Helm can be configured by passing a values YAML file:
+
+```sh
+helm install theodolite theodolite/theodolite --values <your-config.yaml>
+```
+
+For this purpose the [default values file](https://github.com/cau-se/theodolite/blob/master/helm/values.yaml) can serve as a template for your custom configuration.
+
+### Minimal setup
+
+For Kubernetes clusters with limited resources such as on local developer installations, we provide a [minimal values file](https://github.com/cau-se/theodolite/blob/master/helm/preconfigs/minimal.yaml).
+
+### Persisting results
+
+To store the results of benchmark executions in a [PersistentVolume](https://kubernetes.io/docs/concepts/storage/persistent-volumes), `operator.resultsVolume.persistent.enabled` has to be set to `true`. This requires that either a statically provisioned PersistentVolume is available or a dynamic provisioner exists (which is the case for many Kubernetes installations). If required, you can select a storage class with `operator.resultsVolume.persistent.storageClassName`.
+You can also use an existing PersistentVolumeClaim by setting `operator.resultsVolume.persistent.existingClaim`.
+If persistence is not enabled, all results will be gone upon pod termination.
+
+
+### Standalone mode
+
+Per default, Theodolite is installed in operator mode, which allows to run and manage benchmarks through the Kubernetes API. For running Theodolite in standalone mode, it is sufficient to disable the operator by setting `operator.enabled` to `false`. Additionally, you might want to add the command line argument `--skip-crds`. With these settings, only Theodolite's dependencies as well as resources to get the necessary permissions are installed.
+
+### Random scheduler
+
+Installation of the random scheduler can be enabled via `randomScheduler.enabled`. Please note that the random scheduler is neither required in operator mode nor in standalone mode. However, it has to be installed if benchmark executions should use random scheduling.
+
+<!-- **TODO:** link-->
+
+### Multiple installations in the same cluster
+
+Multiple Theodolite installations in the same namespace are currently not fully tested.
+In cases, where you need to install multiple Theodolite instances, it's best to use dedicated namespaces **and** different release names.
+
+*Note that for meaningful results, usually only one benchmark should be executed at a time.*
+
+### Installation with a release name other than `theodolite`
+
+When using another release name than `theodolite`, make sure to adjust the Kafka Lag Exporter configuration of you `values.yaml` accordingly:
+
+```yaml
+kafka-lag-exporter:
+  clusters:
+  - name: "<your-release-name>-cp-kafka"
+    bootstrapBrokers: "<your-release-name>-cp-kafka:9092"
+```
+
+This seems unfortunately to be necessary as Helm does not let us inject values into dependency charts.
+
+
+## Test the Installation
+
+You can test the installation with:
+
+```sh
+helm test theodolite
+```
+
+
+## Uninstall Theodolite
+
+The Theodolite Helm chart can easily be removed with:
+
+```sh
+helm uninstall theodolite
+```
+
+Helm does not remove any CRDs created by this chart. You can remove them manually with:
+
+```sh
+# CRDs for Theodolite
+kubectl delete crd executions.theodolite.com
+kubectl delete crd benchmarks.theodolite.com
+# CRDs for Prometheus operator (see https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack#uninstall-chart)
+kubectl delete crd alertmanagerconfigs.monitoring.coreos.com
+kubectl delete crd alertmanagers.monitoring.coreos.com
+kubectl delete crd podmonitors.monitoring.coreos.com
+kubectl delete crd probes.monitoring.coreos.com
+kubectl delete crd prometheuses.monitoring.coreos.com
+kubectl delete crd prometheusrules.monitoring.coreos.com
+kubectl delete crd servicemonitors.monitoring.coreos.com
+kubectl delete crd thanosrulers.monitoring.coreos.com
+```