diff --git a/execution/README.md b/execution/README.md
index a77eecd20e0c30a98ee1fb5af4496d653458e59e..7b42a3d9729b02cd1c3758e8d2d502211f4b299f 100644
--- a/execution/README.md
+++ b/execution/README.md
@@ -208,6 +208,7 @@ kubectl delete -f theodolite.yaml
| --prometheus | PROMETHEUS_BASE_URL | Defines where to find the prometheus instance. *Default:* `http://localhost:9090` |
| --path | RESULT_PATH | A directory path for the results. Relative to the Execution folder. *Default:* `results` |
| --configurations | CONFIGURATIONS | Defines environment variables for the use cases and, thus, enables further configuration options. |
+| --threshold | THRESHOLD | The threshold for the trend slop that the search strategies use to determine that a load could be handled. *Default:* `2000` |
### Domain Restriction
diff --git a/execution/lib/cli_parser.py b/execution/lib/cli_parser.py
index eaebaa6cc99959bc8a41e50f3d6a63acaf5ab817..de609bc55e21e9467a2b28168be6e478171cfddd 100644
--- a/execution/lib/cli_parser.py
+++ b/execution/lib/cli_parser.py
@@ -136,6 +136,11 @@ def benchmark_parser(description):
metavar='<strategy>',
default=os.environ.get('SEARCH_STRATEGY', 'default'),
help='The benchmarking search strategy. Can be set to default, linear-search or binary-search')
+ parser.add_argument('--threshold',
+ type=int,
+ metavar='<threshold>',
+ default=os.environ.get('THRESHOLD', 2000),
+ help='The threshold for the trend slop that the search strategies use to determine that a load could be handled')
return parser
diff --git a/execution/lib/trend_slope_computer.py b/execution/lib/trend_slope_computer.py
index 294226c35c0038a01804f7f5e8eb3a1e53c79b79..90ae26cfd275f53307e19532f047e5e0a9326d3a 100644
--- a/execution/lib/trend_slope_computer.py
+++ b/execution/lib/trend_slope_computer.py
@@ -2,7 +2,7 @@ from sklearn.linear_model import LinearRegression
import pandas as pd
import os
-def compute(directory, filename, warmup_sec, threshold):
+def compute(directory, filename, warmup_sec):
df = pd.read_csv(os.path.join(directory, filename))
input = df
input['sec_start'] = input.loc[0:, 'timestamp'] - input.iloc[0]['timestamp']
@@ -16,4 +16,4 @@ def compute(directory, filename, warmup_sec, threshold):
trend_slope = linear_regressor.coef_[0][0]
- return trend_slope
\ No newline at end of file
+ return trend_slope
diff --git a/execution/strategies/config.py b/execution/strategies/config.py
index c3cd1ff82c4926f5efcc741b027996dbc800916b..d4df97c18ae54c7c181ddf08264c013f9447350f 100644
--- a/execution/strategies/config.py
+++ b/execution/strategies/config.py
@@ -18,5 +18,6 @@ class ExperimentConfig:
configurations: dict
domain_restriction_strategy: object
search_strategy: object
+ threshold: int
subexperiment_executor: object
subexperiment_evaluator: object
diff --git a/execution/strategies/strategies/search/binary_search_strategy.py b/execution/strategies/strategies/search/binary_search_strategy.py
index 8856ead0502279f8f8642da87cf56f794cb1b11c..46748cbda250597b3a7644522126268be4599293 100644
--- a/execution/strategies/strategies/search/binary_search_strategy.py
+++ b/execution/strategies/strategies/search/binary_search_strategy.py
@@ -7,8 +7,9 @@ def binary_search(config, dim_value, lower, upper, subexperiment_counter):
print(f"Run subexperiment {subexperiment_counter} with config {dim_value} {config.replicass[lower]}")
subexperiment_config = SubexperimentConfig(config.use_case, config.exp_id, subexperiment_counter, dim_value, config.replicass[lower], config.partitions, config.cpu_limit, config.memory_limit, config.execution_minutes, config.prometheus_base_url, config.reset, config.namespace, config.result_path, config.configurations)
config.subexperiment_executor.execute(subexperiment_config)
- result = config.subexperiment_evaluator.execute(subexperiment_config)
- if result==1: # successful, the upper neighbor is assumed to also has been successful
+ success = config.subexperiment_evaluator.execute(subexperiment_config,
+ config.threshold)
+ if success: # successful, the upper neighbor is assumed to also has been successful
return (lower, subexperiment_counter+1)
else: # not successful
return (lower+1, subexperiment_counter)
@@ -16,15 +17,17 @@ def binary_search(config, dim_value, lower, upper, subexperiment_counter):
print(f"Run subexperiment {subexperiment_counter} with config {dim_value} {config.replicass[lower]}")
subexperiment_config = SubexperimentConfig(config.use_case, config.exp_id, subexperiment_counter, dim_value, config.replicass[lower], config.partitions, config.cpu_limit, config.memory_limit, config.execution_minutes, config.prometheus_base_url, config.reset, config.namespace, config.result_path, config.configurations)
config.subexperiment_executor.execute(subexperiment_config)
- result = config.subexperiment_evaluator.execute(subexperiment_config)
- if result==1: # minimal instances found
+ success = config.subexperiment_evaluator.execute(subexperiment_config,
+ config.threshold)
+ if success: # minimal instances found
return (lower, subexperiment_counter)
else: # not successful, check if lower+1 instances are sufficient
print(f"Run subexperiment {subexperiment_counter} with config {dim_value} {config.replicass[upper]}")
subexperiment_config = SubexperimentConfig(config.use_case, config.exp_id, subexperiment_counter, dim_value, config.replicass[upper], config.partitions, config.cpu_limit, config.memory_limit, config.execution_minutes, config.prometheus_base_url, config.reset, config.namespace, config.result_path, config.configurations)
config.subexperiment_executor.execute(subexperiment_config)
- result = config.subexperiment_evaluator.execute(subexperiment_config)
- if result == 1: # minimal instances found
+ success = config.subexperiment_evaluator.execute(subexperiment_config,
+ config.threshold)
+ if success: # minimal instances found
return (upper, subexperiment_counter)
else:
return (upper+1, subexperiment_counter)
@@ -34,8 +37,9 @@ def binary_search(config, dim_value, lower, upper, subexperiment_counter):
print(f"Run subexperiment {subexperiment_counter} with config {dim_value} {config.replicass[mid]}")
subexperiment_config = SubexperimentConfig(config.use_case, config.exp_id, subexperiment_counter, dim_value, config.replicass[mid], config.partitions, config.cpu_limit, config.memory_limit, config.execution_minutes, config.prometheus_base_url, config.reset, config.namespace, config.result_path, config.configurations)
config.subexperiment_executor.execute(subexperiment_config)
- result = config.subexperiment_evaluator.execute(subexperiment_config)
- if result == 1: # success -> search in (lower, mid-1)
+ success = config.subexperiment_evaluator.execute(subexperiment_config,
+ config.threshold)
+ if success: # success -> search in (lower, mid-1)
return binary_search(config, dim_value, lower, mid-1, subexperiment_counter+1)
else: # not success -> search in (mid+1, upper)
return binary_search(config, dim_value, mid+1, upper, subexperiment_counter+1)
diff --git a/execution/strategies/strategies/search/check_all_strategy.py b/execution/strategies/strategies/search/check_all_strategy.py
index 8e9d6c3ca0924d724c4f55032ebc24a92bc3ad93..0861945113b829fa79317d8a1a6312b4d6e4f71d 100644
--- a/execution/strategies/strategies/search/check_all_strategy.py
+++ b/execution/strategies/strategies/search/check_all_strategy.py
@@ -2,23 +2,30 @@
import os
from strategies.strategies.config import SubexperimentConfig
+
def execute(config, dim_value_index, lower_replicas_bound_index, subexperiment_counter):
- new_lower_replicas_bound_index=lower_replicas_bound_index
- new_lower_replicas_bound_found=False
- subexperiments_total=len(config.dim_values)*len(config.replicass)
+ new_lower_replicas_bound_index = lower_replicas_bound_index
+ new_lower_replicas_bound_found = False
+ subexperiments_total = len(config.dim_values) * len(config.replicass)
while lower_replicas_bound_index < len(config.replicass):
- subexperiment_counter+=1
- dim_value=config.dim_values[dim_value_index]
- replicas=config.replicass[lower_replicas_bound_index]
- print(f"Run subexperiment {subexperiment_counter} of {subexperiments_total} with dimension value {dim_value} and {replicas} replicas.")
+ subexperiment_counter += 1
+ dim_value = config.dim_values[dim_value_index]
+ replicas = config.replicass[lower_replicas_bound_index]
+ print(
+ f"Run subexperiment {subexperiment_counter} of {subexperiments_total} with dimension value {dim_value} and {replicas} replicas.")
- subexperiment_config = SubexperimentConfig(config.use_case, config.exp_id, subexperiment_counter, dim_value, replicas, config.partitions, config.cpu_limit, config.memory_limit, config.execution_minutes, config.prometheus_base_url, config.reset, config.namespace, config.result_path, config.configurations)
+ subexperiment_config = SubexperimentConfig(
+ config.use_case, config.exp_id, subexperiment_counter, dim_value,
+ replicas, config.partitions, config.cpu_limit, config.memory_limit,
+ config.execution_minutes, config.prometheus_base_url, config.reset,
+ config.namespace, config.result_path, config.configurations)
config.subexperiment_executor.execute(subexperiment_config)
- result = config.subexperiment_evaluator.execute(subexperiment_config) == 1
- if result == 1 and not new_lower_replicas_bound_found:
+ success = config.subexperiment_evaluator.execute(subexperiment_config,
+ config.threshold)
+ if success and not new_lower_replicas_bound_found:
new_lower_replicas_bound_found = True
new_lower_replicas_bound_index = lower_replicas_bound_index
- lower_replicas_bound_index+=1
+ lower_replicas_bound_index += 1
return (new_lower_replicas_bound_index, subexperiment_counter)
diff --git a/execution/strategies/strategies/search/linear_search_strategy.py b/execution/strategies/strategies/search/linear_search_strategy.py
index f2436658eec0bd4160259a09c272def40fbc130c..8e777303742e54cf2a11a1bde60e95b8aa85489d 100644
--- a/execution/strategies/strategies/search/linear_search_strategy.py
+++ b/execution/strategies/strategies/search/linear_search_strategy.py
@@ -14,8 +14,9 @@ def execute(config, dim_value_index, lower_replicas_bound_index, subexperiment_c
subexperiment_config = SubexperimentConfig(config.use_case, config.exp_id, subexperiment_counter, dim_value, replicas, config.partitions, config.cpu_limit, config.memory_limit, config.execution_minutes, config.prometheus_base_url, config.reset, config.namespace, config.result_path, config.configurations)
config.subexperiment_executor.execute(subexperiment_config)
- result = config.subexperiment_evaluator.execute(subexperiment_config)
- if result == 1:
+ success = config.subexperiment_evaluator.execute(subexperiment_config,
+ config.threshold)
+ if success:
return (lower_replicas_bound_index, subexperiment_counter)
else:
lower_replicas_bound_index+=1
diff --git a/execution/strategies/subexperiment_evaluation/subexperiment_evaluator.py b/execution/strategies/subexperiment_evaluation/subexperiment_evaluator.py
index 442a7999cc0c50257f27271e3c41e6c85c0cecee..30188de837746b76113ec635ca77fadc3a91cb92 100644
--- a/execution/strategies/subexperiment_evaluation/subexperiment_evaluator.py
+++ b/execution/strategies/subexperiment_evaluation/subexperiment_evaluator.py
@@ -1,25 +1,29 @@
import lib.trend_slope_computer as trend_slope_computer
import logging
import os
-import sys
-THRESHOLD = 2000
WARMUP_SEC = 60
-def execute(config):
+def execute(config, threshold):
+ """
+ Check the trend slope of the totallag of the subexperiment if it comes below
+ the threshold.
+
+ :param config: Configuration of the subexperiment.
+ :param threshold: The threshold the trendslope need to come below.
+ """
cwd = f'{os.getcwd()}/{config.result_path}'
file = f"exp{config.exp_id}_uc{config.use_case}_{config.dim_value}_{config.replicas}_totallag.csv"
try:
- trend_slope = trend_slope_computer.compute(cwd, file, WARMUP_SEC, THRESHOLD)
+ trend_slope = trend_slope_computer.compute(cwd, file, WARMUP_SEC)
except Exception as e:
err_msg = 'Computing trend slope failed'
print(err_msg)
logging.exception(err_msg)
print('Mark this subexperiment as not successful and continue benchmark')
- return 0
-
+ return False
print(f"Trend Slope: {trend_slope}")
- success = 0 if trend_slope > THRESHOLD else 1
- return success
+
+ return trend_slope < threshold
diff --git a/execution/theodolite.py b/execution/theodolite.py
index 93717a2c90d874a90b11a4d3c6273b5ca1702687..bd273c4405e2a406b5b5537e084957625c19aa96 100755
--- a/execution/theodolite.py
+++ b/execution/theodolite.py
@@ -31,8 +31,8 @@ def load_variables():
def main(uc, loads, instances_list, partitions, cpu_limit, memory_limit,
- duration, domain_restriction, search_strategy, prometheus_base_url,
- reset, namespace, result_path, configurations):
+ duration, domain_restriction, search_strategy, threshold,
+ prometheus_base_url, reset, namespace, result_path, configurations):
print(
f"Domain restriction of search space activated: {domain_restriction}")
@@ -107,6 +107,7 @@ def main(uc, loads, instances_list, partitions, cpu_limit, memory_limit,
result_path=result_path,
domain_restriction_strategy=domain_restriction_strategy,
search_strategy=search_strategy_method,
+ threshold=threshold,
subexperiment_executor=subexperiment_executor,
subexperiment_evaluator=subexperiment_evaluator)
@@ -119,10 +120,11 @@ if __name__ == '__main__':
args = load_variables()
if args.reset_only:
print('Only reset the cluster')
- run_uc.main(None, None, None, None, None, None, None, None,
- None, None, args.namespace, None, None, reset_only=True)
+ run_uc.main(None, None, None, None, None, None, None, None, None,
+ None, args.namespace, None, None, reset_only=True)
else:
main(args.uc, args.loads, args.instances_list, args.partitions,
args.cpu_limit, args.memory_limit, args.duration,
- args.domain_restriction, args.search_strategy, args.prometheus,
- args.reset, args.namespace, args.path, args.configurations)
+ args.domain_restriction, args.search_strategy,
+ args.threshold, args.prometheus, args.reset, args.namespace,
+ args.path, args.configurations)