Add generic SLO checker

slo-checker/generic/Dockerfile

+FROM tiangolo/uvicorn-gunicorn-fastapi:python3.7
+
+COPY requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+
+COPY ./app /app
\ No newline at end of file

slo-checker/generic/README.md

+# Generic SLO Evaluator
+
+## Execution
+
+For development:
+
+```sh
+uvicorn main:app --reload
+```
+
+## Build the docker image:
+
+```sh
+docker build . -t theodolite-evaluator
+```
+
+Run the Docker image:
+
+```sh
+docker run -p 80:80 theodolite-evaluator
+```
+
+## Configuration
+
+You can set the `HOST` and the `PORT` (and a lot of more parameters) via environment variables. Default is `0.0.0.0:80`.
+For more information see the [Gunicorn/FastAPI Docker docs](https://github.com/tiangolo/uvicorn-gunicorn-fastapi-docker#advanced-usage).
+
+## API Documentation
+
+The running webserver provides a REST API with the following route:
+
+* /
+  * Method: POST
+  * Body:
+    * results
+      * metric-metadata
+      * values
+    * metadata
+      * warmup
+      * queryAggregation
+      * repetitionAggregation
+      * operator
+      * threshold
+
+The body of the request must be a JSON string that satisfies the following conditions:
+
+* **dropped records**: This property is based on the [Range Vector type](https://www.prometheus.io/docs/prometheus/latest/querying/api/#range-vectors) from Prometheus and must have the following JSON *structure*:
+
+    ```json
+    {
+        "results": [
+            [
+                {
+                    "metric": {
+                        "<label-name>": "<label-value>"
+                    },
+                    "values": [
+                        [
+                            <unix_timestamp>, // 1.634624989695E9
+                            "<sample_value>" // integer
+                        ]
+                    ]
+                }
+            ]
+        ],
+        "metadata": {
+            "warmup": 60,
+            "queryAggregation": "max",
+            "repetitionAggregation": "median",
+            "operator": "lt",
+            "threshold": 2000000
+        }
+    }
+    ```
+
+### description
+
+* results:
+  * metric-metadata:
+    * Labels of this metric. The `generic` slo checker does not use labels in the calculation of the service level objective.
+  * results
+    * The `<unix_timestamp>` provided as the first element of each element in the "values" array must be the timestamp of the measurement value in seconds (with optional decimal precision)
+    * The `<sample_value>` must be the measurement value as string.
+* metadata: For the calculation of the service level objective require metadata.
+  * **warmup**: Specifies the warmup time in seconds that are ignored for evaluating the SLO.
+  * **queryAggregation**: Specifies the function used to aggregate a query. 
+  * **repetitionAggregation**: Specifies the function used to aggregate a the results of multiple query aggregations.
+  * **operator**: Specifies how the result should be checked agains a threshold. Possible values are `lt`, `lte`, `gt` and `gte`.
+  * **threshold**: Must be an unsigned integer that specifies the threshold for the SLO evaluation.

slo-checker/generic/app/main.py

+from fastapi import FastAPI,Request
+import logging
+import os
+import json
+import sys
+import re
+import pandas as pd
+
+
+app = FastAPI()
+
+logging.basicConfig(stream=sys.stdout,
+                    format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+logger = logging.getLogger("API")
+
+
+if os.getenv('LOG_LEVEL') == 'INFO':
+    logger.setLevel(logging.INFO)
+elif os.getenv('LOG_LEVEL') == 'WARNING':
+    logger.setLevel(logging.WARNING)
+elif os.getenv('LOG_LEVEL') == 'DEBUG':
+    logger.setLevel(logging.DEBUG)
+
+
+def get_aggr_func(func_string: str):
+    if func_string in ['mean', 'median', 'mode', 'sum', 'count', 'max', 'min', 'std', 'var', 'skew', 'kurt']:
+        return func_string
+    elif re.search(r'^p\d\d?(\.\d+)?$', func_string): # matches strings like 'p99', 'p99.99', 'p1', 'p0.001'
+        def percentile(x):
+            return x.quantile(float(func_string[1:]) / 100)
+        percentile.__name__ = func_string
+        return percentile
+    else:
+        raise ValueError('Invalid function string.')
+
+def aggr_query(values: dict, warmup: int, aggr_func):
+    df = pd.DataFrame.from_dict(values)
+    df.columns = ['timestamp', 'value']
+    filtered = df[df['timestamp'] >= (df['timestamp'][0] + warmup)]
+    filtered['value'] = filtered['value'].astype(int)
+    return filtered['value'].aggregate(aggr_func)
+
+def check_result(result, operator: str, threshold):
+    if operator == 'lt':
+        return result < threshold
+    if operator == 'lte':
+        return result <= threshold
+    if operator == 'gt':
+        return result > threshold
+    if operator == 'gte':
+        return result >= threshold
+    else:
+        raise ValueError('Invalid operator string.')
+
+
+
+@app.post("/",response_model=bool)
+async def check_slo(request: Request):
+    data = json.loads(await request.body())
+    warmup = int(data['metadata']['warmup'])
+    query_aggregation = get_aggr_func(data['metadata']['queryAggregation'])
+    rep_aggregation = get_aggr_func(data['metadata']['repetitionAggregation'])
+    operator = data['metadata']['operator']
+    threshold = int(data['metadata']['threshold'])
+
+    for r in data["results"]:
+        aggr_query(r[0]["values"], warmup, query_aggregation)
+
+    query_results = [aggr_query(r[0]["values"], warmup, query_aggregation) for r in data["results"]]
+    result = pd.DataFrame(query_results).aggregate(rep_aggregation).at[0]
+    return check_result(result, operator, threshold)
+
+logger.info("SLO evaluator is online")
\ No newline at end of file

slo-checker/generic/app/test.py

+import unittest
+from main import app, get_aggr_func, check_result
+import json
+from fastapi.testclient import TestClient
+
+class TestSloEvaluation(unittest.TestCase):
+    client = TestClient(app)
+
+    def test_1_rep(self):
+        with open('../resources/test-1-rep-success.json') as json_file:
+            data = json.load(json_file)
+            response = self.client.post("/", json=data)
+            self.assertEqual(response.json(), True)
+
+    def test_get_aggr_func_mean(self):
+        self.assertEqual(get_aggr_func('median'), 'median')
+    
+    def test_get_aggr_func_p99(self):
+        self.assertTrue(callable(get_aggr_func('p99')))
+
+    def test_get_aggr_func_p99_9(self):
+        self.assertTrue(callable(get_aggr_func('p99.9')))
+
+    def test_get_aggr_func_p99_99(self):
+        self.assertTrue(callable(get_aggr_func('p99.99')))
+
+    def test_get_aggr_func_p0_1(self):
+        self.assertTrue(callable(get_aggr_func('p0.1')))
+
+    def test_get_aggr_func_p99_(self):
+        self.assertRaises(ValueError, get_aggr_func, 'p99.')
+
+    def test_get_aggr_func_p99_(self):
+        self.assertRaises(ValueError, get_aggr_func, 'q99')
+
+    def test_get_aggr_func_p99_(self):
+        self.assertRaises(ValueError, get_aggr_func, 'mux')
+    
+    def test_check_result_lt(self):
+        self.assertEqual(check_result(100, 'lt', 200), True)
+        
+    def test_check_result_lte(self):
+        self.assertEqual(check_result(200, 'lte', 200), True)
+    
+    def test_check_result_gt(self):
+        self.assertEqual(check_result(100, 'gt', 200), False)
+
+    def test_check_result_gte(self):
+        self.assertEqual(check_result(300, 'gte', 200), True)
+
+    def test_check_result_invalid(self):
+        self.assertRaises(ValueError, check_result, 100, 'xyz', 200)
+
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

slo-checker/generic/requirements.txt

+fastapi==0.65.2
+pandas==1.0.3
+uvicorn
+requests

slo-checker/generic/resources/test-1-rep-success.json

+{
+    "results": [
+        [
+            {
+                "metric": {
+                    "job": "titan-ccp-aggregation"
+                },
+                "values": [
+                    [
+                        1.634624674695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624679695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624684695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624689695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624694695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624699695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624704695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624709695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624714695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624719695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624724695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624729695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624734695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624739695E9,
+                        "0"
+                    ],
+                    [
+                        1.634624744695E9,
+                        "1"
+                    ],
+                    [
+                        1.634624749695E9,
+                        "3"
+                    ],
+                    [
+                        1.634624754695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624759695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624764695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624769695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624774695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624779695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624784695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624789695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624794695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624799695E9,
+                        "4"
+                    ],
+                    [
+                        1.634624804695E9,
+                        "176"
+                    ],
+                    [
+                        1.634624809695E9,
+                        "176"
+                    ],
+                    [
+                        1.634624814695E9,
+                        "176"
+                    ],
+                    [
+                        1.634624819695E9,
+                        "176"
+                    ],
+                    [
+                        1.634624824695E9,
+                        "176"
+                    ],
+                    [
+                        1.634624829695E9,
+                        "159524"
+                    ],
+                    [
+                        1.634624834695E9,
+                        "209870"
+                    ],
+                    [
+                        1.634624839695E9,
+                        "278597"
+                    ],
+                    [
+                        1.634624844695E9,
+                        "460761"
+                    ],
+                    [
+                        1.634624849695E9,
+                        "460761"
+                    ],
+                    [
+                        1.634624854695E9,
+                        "460761"
+                    ],
+                    [
+                        1.634624859695E9,
+                        "460761"
+                    ],
+                    [
+                        1.634624864695E9,
+                        "460761"
+                    ],
+                    [
+                        1.634624869695E9,
+                        "606893"
+                    ],
+                    [
+                        1.634624874695E9,
+                        "653534"
+                    ],
+                    [
+                        1.634624879695E9,
+                        "755796"
+                    ],
+                    [
+                        1.634624884695E9,
+                        "919317"
+                    ],
+                    [
+                        1.634624889695E9,
+                        "919317"
+                    ],
+                    [
+                        1.634624894695E9,
+                        "955926"
+                    ],
+                    [
+                        1.634624899695E9,
+                        "955926"
+                    ],
+                    [
+                        1.634624904695E9,
+                        "955926"
+                    ],
+                    [
+                        1.634624909695E9,
+                        "955926"
+                    ],
+                    [
+                        1.634624914695E9,
+                        "955926"
+                    ],
+                    [
+                        1.634624919695E9,
+                        "1036530"
+                    ],
+                    [
+                        1.634624924695E9,
+                        "1078477"
+                    ],
+                    [
+                        1.634624929695E9,
+                        "1194775"
+                    ],
+                    [
+                        1.634624934695E9,
+                        "1347755"
+                    ],
+                    [
+                        1.634624939695E9,
+                        "1352151"
+                    ],
+                    [
+                        1.634624944695E9,
+                        "1360428"
+                    ],
+                    [
+                        1.634624949695E9,
+                        "1360428"
+                    ],
+                    [
+                        1.634624954695E9,
+                        "1360428"
+                    ],
+                    [
+                        1.634624959695E9,
+                        "1360428"
+                    ],
+                    [
+                        1.634624964695E9,
+                        "1360428"
+                    ],
+                    [
+                        1.634624969695E9,
+                        "1525685"
+                    ],
+                    [
+                        1.634624974695E9,
+                        "1689296"
+                    ],
+                    [
+                        1.634624979695E9,
+                        "1771358"
+                    ],
+                    [
+                        1.634624984695E9,
+                        "1854284"
+                    ],
+                    [
+                        1.634624989695E9,
+                        "1854284"
+                    ]
+                ]
+            }
+        ]
+    ],
+    "metadata": {
+        "warmup": 60,
+        "queryAggregation": "max",
+        "repetitionAggregation": "median",
+        "operator": "lt",
+        "threshold": 2000000
+    }
+}
\ No newline at end of file