diff --git a/.gitignore b/.gitignore index 69aa4917240ab6f4262f7f474fa9a65fc77ef80d..9d9d686e6517a08446a51b864dcc0df9a8e8e7e7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ **/.settings/** **/build/** **/bin/** +frameworks/log_* +results/ frameworks/Kieker/scripts/benchmark/ frameworks/Kieker/scripts/data/ frameworks/Kieker/MooBench.jar diff --git a/README.md b/README.md index 3646a59ed36333f2cbe1dc37c780ef00ca3855ba..af185c1860698e9d0facc3d80f53e2a40b90a2ee 100644 --- a/README.md +++ b/README.md @@ -48,4 +48,6 @@ cd frameworks/opentelemetry/ ## Data Analysis Each benchmark execution calls an R script providing mean, standard deviation and confidence intervals for the benchmark variants. If you want to get these values again, switch to `frameworks` and call `runR.sh $FRAMEWORK`, where framework is the folder name of the framework (e.g. Kieker). +If you got data from a run with exponential growing call tree depth, unzip them first (`for file in *.zip; do unzip $file; done`), copy all `results-$framework` folder to a common folder and run `./getExponential.sh` in analysis. This will create a graph for each framework and an overview graph for external processing of the traces (zipkin for OpenTelemetry and inspectIT, TCP for Kieker). + In the folder /bin/r are some R scripts provided to generate graphs to visualize the results. In the top the files, one can configure the required paths and the configuration used to analyze the data. diff --git a/analysis/getExponential.sh b/analysis/getExponential.sh new file mode 100755 index 0000000000000000000000000000000000000000..95164d635de10fa22a5a05033ac8dc6aaba7738e --- /dev/null +++ b/analysis/getExponential.sh @@ -0,0 +1,62 @@ +function getSum { + awk '{sum += $1; square += $1^2} END {print "Average: "sum/NR" Standard Deviation: "sqrt(square / NR - (sum/NR)^2)" Count: "NR}' +} + +function getFileAverages { + variants=$(ls $1 | grep raw | awk -F'[-.]' '{print $4}' | sort | uniq) + for size in 2 4 8 16 32 64 128 + do + for variant in $variants + do + allExecutions=$(cat $1/raw-*-$size-$variant.csv | wc -l) + for file in $(ls $1/raw-*-$size-$variant.csv) + do + fileSize=$(cat $file | wc -l) + afterWarmup=$(($fileSize/2)) + average=$(tail -n $afterWarmup $file | awk -F';' '{print $2}' | getSum | awk '{print $2}') + echo $variant";"$size";"$average + done + done + done +} + +function getFrameworkEvolutionFile { + folder=$1 + framework=$2 + getFileAverages $1/results-$framework/ > $RESULTFOLDER/$framework.csv + variants=$(cat $RESULTFOLDER/$framework.csv | awk -F';' '{print $1}' | sort | uniq) + for size in 2 4 8 16 32 64 128 + do + echo -n "$size;" + for variant in $variants + do + cat $RESULTFOLDER/$framework.csv | grep "^$variant;$size;" | awk -F';' '{print $3}' | getSum | awk '{print $2";"$5";"}' | tr -d "\n" + done + echo + done > $RESULTFOLDER/evolution_$framework.csv +} + +if [ "$#" -lt 1 ]; then + echo "Please pass the folder where results-Kieker, results-OpenTelemetry and results-inspectIT are" + exit 1 +fi + +if [ ! -d $1 ]; then + echo "$1 should be a folder, but is not." + exit 1 +fi + +RESULTFOLDER=../results +if [ -d $RESULTFOLDER ] +then + rm -rf $RESULTFOLDER/* +fi +mkdir -p $RESULTFOLDER + +for framework in Kieker OpenTelemetry inspectIT +do + echo "Analysing $framework" + getFrameworkEvolutionFile $1 $framework +done + +gnuplot -c plotExponential.plt diff --git a/analysis/getFileAverages.sh b/analysis/getFileAverages.sh new file mode 100755 index 0000000000000000000000000000000000000000..0f0f85342e8b979ed9b73eee7e39a7f55a0fc073 --- /dev/null +++ b/analysis/getFileAverages.sh @@ -0,0 +1,19 @@ +function getSum { + awk '{sum += $1; square += $1^2} END {print "Average: "sum/NR" Standard Deviation: "sqrt(square / NR - (sum/NR)^2)" Count: "NR}' +} + +variants=$(ls $1 | grep raw | awk -F'[-.]' '{print $4}' | sort | uniq) +for size in 2 4 8 16 32 64 128 +do + for variant in $variants + do + allExecutions=$(cat $1/raw-*-$size-$variant.csv | wc -l) + for file in $(ls $1/raw-*-$size-$variant.csv) + do + fileSize=$(cat $file | wc -l) + afterWarmup=$(($fileSize/2)) + average=$(tail -n $afterWarmup $file | awk -F';' '{print $2}' | getSum | awk '{print $2}') + echo $variant";"$size";"$average + done + done +done diff --git a/analysis/plotExponential.plt b/analysis/plotExponential.plt new file mode 100644 index 0000000000000000000000000000000000000000..896505c02e856414f64d22021cfe95dfd4d3c5a8 --- /dev/null +++ b/analysis/plotExponential.plt @@ -0,0 +1,93 @@ +set encoding iso_8859_1 +set terminal pdf size 8,5 + +set datafile separator ";" + +set out '../results/Kieker.pdf' + +set title 'Kieker Method Execution Durations' + +set xlabel 'Call Tree Depth' +set ylabel 'Duration {/Symbol m}s' + +set key right center + +plot '../results/evolution_Kieker.csv' u 1:2 w linespoint lc "red" title 'Baseline', \ + '../results/evolution_Kieker.csv' u 1:($2-$3):($2+$3) w filledcurves lc "red" notitle fs transparent solid 0.5, \ + '../results/evolution_Kieker.csv' u 1:4 w linespoint lc "yellow" title 'Deactivated Probe', \ + '../results/evolution_Kieker.csv' u 1:($4-$5):($4+$5) w filledcurves lc "yellow" notitle fs transparent solid 0.5, \ + '../results/evolution_Kieker.csv' u 1:10 w linespoint lc "red" title 'Logging (Binary)', \ + '../results/evolution_Kieker.csv' u 1:($10-$11):($10+$11) w filledcurves lc "red" notitle fs transparent solid 0.5, \ + '../results/evolution_Kieker.csv' u 1:12 w linespoint lc "blue" title 'TCP', \ + '../results/evolution_Kieker.csv' u 1:($12-$13):($12+$13) w filledcurves lc "blue" notitle fs transparent solid 0.5 +# '../results/evolution_Kieker.csv' u 1:8 w linespoint lc "green" title 'Logging (Text)', \ +# '../results/evolution_Kieker.csv' u 1:($8-$9):($8+$9) w filledcurves lc "green" notitle fs transparent solid 0.5, \ +# Activate this, if text logging should be displayed (very big, so disabled by default) + +unset output + +set out '../results/OpenTelemetry.pdf' + +set title 'OpenTelemetry Method Execution Durations' + +set xlabel 'Call Tree Depth' +set ylabel 'Duration {/Symbol m}s' + +set key right center + +plot '../results/evolution_OpenTelemetry.csv' u 1:2 w linespoint lc "red" title 'Baseline', \ + '../results/evolution_OpenTelemetry.csv' u 1:($2-$3):($2+$3) w filledcurves lc "red" notitle fs transparent solid 0.5, \ + '../results/evolution_OpenTelemetry.csv' u 1:4 w linespoint lc "yellow" title 'No Logging', \ + '../results/evolution_OpenTelemetry.csv' u 1:($4-$5):($4+$5) w filledcurves lc "yellow" notitle fs transparent solid 0.5, \ + '../results/evolution_OpenTelemetry.csv' u 1:8 w linespoint lc "red" title 'Zipkin', \ + '../results/evolution_OpenTelemetry.csv' u 1:($8-$9):($8+$9) w filledcurves lc "red" notitle fs transparent solid 0.5, \ + '../results/evolution_OpenTelemetry.csv' u 1:10 w linespoint lc "blue" title 'Prometheus', \ + '../results/evolution_OpenTelemetry.csv' u 1:($10-$11):($10+$11) w filledcurves lc "blue" notitle fs transparent solid 0.5 + +# 'evolution_OpenTelemetry.csv' u 1:6 w linespoint lc "green" title 'Logging (Text)', \ +# 'evolution_OpenTelemetry.csv' u 1:($6-$7):($6+$7) w filledcurves lc "green" notitle fs transparent solid 0.5, \ +# Activate this, if text logging should be displayed (very big, so disabled by default) + +unset output + +set out '../results/inspectIT.pdf' + +set title 'inspectIT Method Execution Durations' + +set xlabel 'Call Tree Depth' +set ylabel 'Duration {/Symbol m}s' + +set key right center + +plot '../results/evolution_inspectIT.csv' u 1:2 w linespoint lc "red" title 'Baseline', \ + '../results/evolution_inspectIT.csv' u 1:($2-$3):($2+$3) w filledcurves lc "red" notitle fs transparent solid 0.5, \ + '../results/evolution_inspectIT.csv' u 1:4 w linespoint lc "yellow" title 'No Logging', \ + '../results/evolution_inspectIT.csv' u 1:($4-$5):($4+$5) w filledcurves lc "yellow" notitle fs transparent solid 0.5, \ + '../results/evolution_inspectIT.csv' u 1:6 w linespoint lc "red" title 'Zipkin', \ + '../results/evolution_inspectIT.csv' u 1:($6-$7):($6+$7) w filledcurves lc "red" notitle fs transparent solid 0.5, \ + '../results/evolution_inspectIT.csv' u 1:8 w linespoint lc "blue" title 'Prometheus', \ + '../results/evolution_inspectIT.csv' u 1:($8-$9):($8+$9) w filledcurves lc "blue" notitle fs transparent solid 0.5 + + +unset output + +set out '../results/overview.pdf' + +set title 'Overview of Method Execution Durations' + +set xlabel 'Call Tree Depth' +set ylabel 'Duration {/Symbol m}s' + +set key right center + +plot '../results/evolution_inspectIT.csv' u 1:2 w linespoint lc "red" title 'Baseline', \ + '../results/evolution_inspectIT.csv' u 1:($2-$3):($2+$3) w filledcurves lc "red" notitle fs transparent solid 0.5, \ + '../results/evolution_Kieker.csv' u 1:12 w linespoint lc "blue" title 'Kieker (TCP)', \ + '../results/evolution_Kieker.csv' u 1:($12-$13):($12+$13) w filledcurves lc "blue" notitle fs transparent solid 0.5, \ + '../results/evolution_inspectIT.csv' u 1:6 w linespoint lc rgb "#c66900" title 'inspectIT (Zipkin)', \ + '../results/evolution_inspectIT.csv' u 1:($6-$7):($6+$7) w filledcurves lc rgb "#c66900" notitle fs transparent solid 0.5, \ + '../results/evolution_OpenTelemetry.csv' u 1:8 w linespoint lc "green" title 'OpenTelemetry (Zipkin)', \ + '../results/evolution_OpenTelemetry.csv' u 1:($8-$9):($8+$9) w filledcurves lc "green" notitle fs transparent solid 0.5 + + +unset output