diff --git a/execution/lag_analysis.py b/execution/lag_analysis.py index f2690ab13b072171020cd5b27a55d6260b9b9084..c529853fd423babf0783331b02810f4e892af357 100644 --- a/execution/lag_analysis.py +++ b/execution/lag_analysis.py @@ -50,6 +50,8 @@ for result in results: df = pd.DataFrame(d) +# Do some analysis + input = df.loc[df['topic'] == "input"] #input.plot(kind='line',x='timestamp',y='value',color='red') @@ -83,6 +85,30 @@ plt.savefig(f"{filename}_plot.png") df.to_csv(f"{filename}_values.csv") +# Load total lag count + +response = requests.get('http://kube1.se.internal:32529/api/v1/query_range', params={ + 'query': "sum by(group)(kafka_consumergroup_group_lag > 0)", + 'start': start.isoformat(), + 'end': end.isoformat(), + 'step': '5s'}) + +results = response.json()['data']['result'] + +d = [] + +for result in results: + #print(result['metric']['topic']) + group = result['metric']['group'] + for value in result['values']: + #print(value) + d.append({'group': group, 'timestamp': int(value[0]), 'value': int(value[1]) if value[1] != 'NaN' else 0}) + +df = pd.DataFrame(d) + +df.to_csv(f"{filename}_totallag.csv") + + # Load partition count response = requests.get('http://kube1.se.internal:32529/api/v1/query_range', params={ diff --git a/execution/scalability-graph.ipynb b/execution/scalability-graph.ipynb index 31fb5c2c991ac0cfe7c141202db597b9630b4d64..9de4b2011462e6c8281147520652db6caf3e8833 100644 --- a/execution/scalability-graph.ipynb +++ b/execution/scalability-graph.ipynb @@ -1,27 +1,4 @@ { - "nbformat": 4, - "nbformat_minor": 2, - "metadata": { - "language_info": { - "name": "python", - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "version": "3.7.0-final" - }, - "orig_nbformat": 2, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "npconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": 3, - "kernelspec": { - "name": "python37064bitvenvvenv469ea2e0a7854dc7b367eee45386afee", - "display_name": "Python 3.7.0 64-bit ('.venv': venv)" - } - }, "cells": [ { "cell_type": "code", @@ -61,10 +38,9 @@ "metadata": {}, "outputs": [], "source": [ - "exp_id = 35\n", + "exp_id = 159\n", "warmup_sec = 60\n", - "threshold = 2000 #slope\n", - "" + "threshold = 2000 #slope\n" ] }, { @@ -84,7 +60,7 @@ "\n", "raw_runs = []\n", "\n", - "filenames = [filename for filename in os.listdir('.') if filename.startswith(f\"exp{exp_id}\") and filename.endswith(\".csv\")]\n", + "filenames = [filename for filename in os.listdir('.') if filename.startswith(f\"exp{exp_id}\") and filename.endswith(\"totallag.csv\")]\n", "for filename in filenames:\n", " #print(filename)\n", " run_params = filename[:-4].split(\"_\")\n", @@ -92,7 +68,8 @@ " instances = run_params[3]\n", "\n", " df = pd.read_csv(filename)\n", - " input = df.loc[df['topic'] == \"input\"]\n", + " #input = df.loc[df['topic'] == \"input\"]\n", + " input = df\n", " #print(input)\n", " input['sec_start'] = input.loc[0:, 'timestamp'] - input.iloc[0]['timestamp']\n", " #print(input)\n", @@ -116,8 +93,7 @@ " #print(row)\n", " raw_runs.append(row)\n", "\n", - "runs = pd.DataFrame(raw_runs)\n", - "" + "runs = pd.DataFrame(raw_runs)\n" ] }, { @@ -176,5 +152,28 @@ "outputs": [], "source": [] } - ] + ], + "metadata": { + "language_info": { + "name": "python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "version": "3.7.0-final" + }, + "orig_nbformat": 2, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "npconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": 3, + "kernelspec": { + "name": "python37064bitvenvvenv469ea2e0a7854dc7b367eee45386afee", + "display_name": "Python 3.7.0 64-bit ('.venv': venv)" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } \ No newline at end of file