diff --git a/.gitignore b/.gitignore index 7b247314b6b47a489fd1f0bd0abc47d1e2a6b8e8..c98caeabdd78e50c471b75240f9be996c798d0d0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ **/bin/** /.gradle/ /build/ +/benchmark/** diff --git a/Jenkinsfile b/Jenkinsfile index fcf00c48b7877cc06907da7dbcf6f5ca3ae757b7..fe46949cc0094b39f53ad592756d29cffa650b9b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ pipeline { stage('Run Benchmark') { steps { - sh '${BASE_DIR}/run-benchmark.sh ${KEYSTORE} ${UPDATE_SITE_URL}' + sh '${BASE_DIR}/run-all.sh ${KEYSTORE} ${UPDATE_SITE_URL}' } } diff --git a/README.md b/README.md index 26c8e8696e6823bd2a2de4e509556580b5f1fafd..398603026ddacf58d9bf124e9b966a55ade0b0e3 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,9 @@ Initially, the following steps are required: 1. Make sure, that you've installed R (http://www.r-project.org/) to generate the results , awk to install intermediate results and curl to download processing tools (Ubuntu: `sudo apt install r-base gawk curl`). -2. Compile the application by calling `./gradlew assemble` in the main folder. +2. Compile the application and install it in the repository root directory. + This can be done automatically be calling + `./setup.sh` All experiments are started with the provided "External Controller" scripts. The following scripts are available for every framework ($FRAMEWORK): diff --git a/build.xml b/build.xml deleted file mode 100644 index f7387d4dbd33aa0694304349dbf7f9101334c64e..0000000000000000000000000000000000000000 --- a/build.xml +++ /dev/null @@ -1,109 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project name="MooBench" default="build-all" basedir="."> - - <target name="build-all" depends="clean,init,compile,jar" /> - <target name="build-kieker" depends="clean,init,compile-kieker,jar" /> - <target name="build-inspectit" depends="clean,init,compile-inspectit,jar" /> - <target name="build-spassmeter" depends="clean,init,compile-spassmeter,jar" /> - <target name="build-kieker-ci" depends="clean,init,compile-kieker,jar,jar-kieker-ci" /> - - <target name="init" depends=""> - <mkdir dir="build" /> - <mkdir dir="dist" /> - <mkdir dir="dist/lib" /> - <mkdir dir="dist/tmp" /> - </target> - - <target name="compile" depends="init"> - <path id="compile-classpath"> - <fileset dir="lib/"> - <include name="commons-cli-*.jar" /> - </fileset> - </path> - <javac source="1.6" target="1.6" destDir="build" debug="true" classpathref="compile-classpath" includeAntRuntime="false"> - <compilerarg value="-Xlint:-options"/> - <src path="src" /> - </javac> - <copy todir="build/META-INF"> - <fileset dir="src/META-INF" /> - </copy> - </target> - - <target name="compile-kieker" depends="compile"> - <path id="compile-classpath"> - <fileset dir="frameworks/Kieker/lib/"> - <include name="kieker-1.11-SNAPSHOT_aspectj.jar" /> - </fileset> - </path> - <javac source="1.6" target="1.6" destDir="build" debug="true" classpathref="compile-classpath" includeAntRuntime="false"> - <compilerarg value="-Xlint:-options"/> - <src path="frameworks/Kieker/src" /> - </javac> - <copy todir="build/META-INF"> - <fileset dir="frameworks/Kieker/src/META-INF" /> - </copy> - <copy todir="dist/lib" flatten="true"> - <path> - <path refid="compile-classpath"/> - </path> - </copy> - <copy file="frameworks/Kieker/bin/benchmark.sh" todir="dist" /> - </target> - - <target name="compile-inspectit" depends="compile"> - <copy todir="dist"> - <fileset dir="frameworks/inspectIT/"> - <exclude name="**/*.launch"/> - </fileset> - </copy> - </target> - - <target name="compile-spassmeter" depends="compile"> - <copy todir="dist"> - <fileset dir="frameworks/SPASSmeter/"> - <exclude name="**/*.launch"/> - </fileset> - </copy> - </target> - - <target name="jar" depends="compile"> - <manifestclasspath property="jar.classpath" jarfile="dist/MooBench.jar"> - <classpath> - <fileset dir="dist/lib/"> - <include name="**/*.jar" /> - <exclude name="**/spass-meter-ant.jar" /> - </fileset> - </classpath> - </manifestclasspath> - <jar destfile="dist/MooBench.jar"> - <fileset dir="build" /> - <fileset dir="." includes="LICENSE README" /> - <manifest> - <attribute name="Main-Class" value="mooBench.benchmark.Benchmark" /> - <attribute name="Class-Path" value="${jar.classpath}" /> - </manifest> - <zipfileset src="lib/commons-cli-1.2.jar" /> - </jar> - <copy todir="dist/r"> - <fileset dir="bin/r" /> - </copy> - </target> - - <target name="jar-kieker-ci" depends="jar"> - <copy file="frameworks/Kieker/bin/benchmark-ci.sh" todir="dist" /> - <mkdir dir="dist/MooBench" /> - <move todir="dist/MooBench"> - <fileset dir="dist"> - <exclude name="MooBench"/> - </fileset> - </move> - <delete file="dist/MooBench/lib/kieker-1.11-SNAPSHOT_aspectj.jar" /> - <copy file="continuous-integration/executeRemoteMicroBenchmark.sh" todir="dist" /> - </target> - - <target name="clean"> - <delete dir="build" /> - <delete dir="dist" /> - </target> - -</project> diff --git a/config.rc b/config.rc new file mode 100644 index 0000000000000000000000000000000000000000..0872ef3c3347a9cb7fe3dcd99b7a684b46e8fe7f --- /dev/null +++ b/config.rc @@ -0,0 +1,5 @@ +## Moobench setup script + +MOOBENCH_ARCHIVE="${BASE_DIR}/tools/benchmark/build/distributions/benchmark.tar" + + diff --git a/frameworks/Kieker/java/benchmark.sh b/frameworks/Kieker/java/benchmark.sh index fa74eb883b227310f50dafad61724b5051582d41..aff8abbe4613c2bef5e597d741310651f1d83046 100755 --- a/frameworks/Kieker/java/benchmark.sh +++ b/frameworks/Kieker/java/benchmark.sh @@ -67,19 +67,16 @@ info "----------------------------------" info "Setup..." info "----------------------------------" +cd "${BASE_DIR}" + # load agent getAgent # Find receiver and extract it - checkFile receiver "${RECEIVER_ARCHIVE}" tar -xpf "${RECEIVER_ARCHIVE}" RECEIVER_BIN="${BASE_DIR}/receiver/bin/receiver" -checkFile moobench "${MOOBENCH_ARCHIVE}" -tar -xpf "${MOOBENCH_ARCHIVE}" -MOOBENCH_BIN="${BASE_DIR}/benchmark/bin/benchmark" - PARENT=`dirname "${RESULTS_DIR}"` checkDirectory DATA_DIR "${DATA_DIR}" create @@ -163,8 +160,7 @@ if [ "$MODE" == "execute" ] ; then # Create R labels LABELS=$(createRLabels) - runRyaml - read R + runStatistics cleanupResults else executeBenchmarkBody $OPTION 1 1 diff --git a/frameworks/Kieker/java/config.rc b/frameworks/Kieker/java/config.rc index b6216f964d6e44b7ccaeed72ec677045374ba412..77b521be934f3fe8e0594b2c729f08b35e78b879 100644 --- a/frameworks/Kieker/java/config.rc +++ b/frameworks/Kieker/java/config.rc @@ -8,8 +8,8 @@ RSCRIPT_PATH="${BASE_DIR}/../../statistics.r" MAIN_DIR="${BASE_DIR}/../../.." +MOOBENCH_BIN="${MAIN_DIR}/benchmark/bin/benchmark" RECEIVER_ARCHIVE="${MAIN_DIR}/tools/receiver/build/distributions/receiver.tar" -MOOBENCH_ARCHIVE="${MAIN_DIR}/tools/benchmark/build/distributions/benchmark.tar" DATA_DIR="${BASE_DIR}/data" AGENT="${BASE_DIR}/kieker-2.0.0-SNAPSHOT-aspectj.jar" diff --git a/frameworks/Kieker/java/functions.sh b/frameworks/Kieker/java/functions.sh index 5aa5682b69d7d0aae43bdcafcf4d9ddff097960c..39c56b7d60a1677d92ec33e45a6a78ca8e3b2dc9 100644 --- a/frameworks/Kieker/java/functions.sh +++ b/frameworks/Kieker/java/functions.sh @@ -7,6 +7,7 @@ then exit 1 fi + function getAgent() { info "Checking whether Kieker is present in ${AGENT}" if [ ! -f "${AGENT}" ] ; then @@ -21,6 +22,8 @@ function getAgent() { fi } +# experiment setups + ################################# # function: execute an experiment # @@ -36,7 +39,7 @@ function executeExperiment() { title="$4" kieker_parameters="$5" - info " # recursion=${recursion} loop=${loop} writer=${index} ${title}" + info " # ${loop}.${recursion}.${index} ${title}" echo " # ${loop}.${recursion}.${index} ${title}" >> "${DATA_DIR}/kieker.log" if [ "${kieker_parameters}" == "" ] ; then diff --git a/frameworks/OpenTelemetry/benchmark.sh b/frameworks/OpenTelemetry/benchmark.sh index 6dd4b687091a99ddf43dfa7ed4202936ca0440de..9539f8778ceed35582fa8e07822f5edd1c2d6328 100755 --- a/frameworks/OpenTelemetry/benchmark.sh +++ b/frameworks/OpenTelemetry/benchmark.sh @@ -53,6 +53,7 @@ info "----------------------------------" info "Setup..." info "----------------------------------" +# load agent getAgent checkExecutable MooBench "${MOOBENCH_BIN}" @@ -67,7 +68,6 @@ TIME=`expr ${METHOD_TIME} \* ${TOTAL_NUM_OF_CALLS} / 1000000000 \* 4 \* ${RECURS info "Experiment will take circa ${TIME} seconds." JAVA_ARGS="-server" -JAVA_ARGS="${JAVA_ARGS} " JAVA_ARGS="${JAVA_ARGS} -Xms1G -Xmx2G" JAVA_ARGS="${JAVA_ARGS} -verbose:gc " @@ -115,7 +115,7 @@ done # Create R labels LABELS=$(createRLabels) -runR +runStatistics cleanupResults @@ -124,4 +124,3 @@ mv "${BASE_DIR}/OpenTelemetry.log" "${RESULTS_DIR}/OpenTelemetry.log" [ -f "${BASE_DIR}/errorlog.txt" ] && mv "${BASE_DIR}/errorlog.txt" "${RESULTS_DIR}" # end - diff --git a/frameworks/OpenTelemetry/functions.sh b/frameworks/OpenTelemetry/functions.sh index d65f2b7748e00c400867cbf780c5a81323383642..0cebbf59c3b12b1eb3d5ccc19d749163b7cc320b 100644 --- a/frameworks/OpenTelemetry/functions.sh +++ b/frameworks/OpenTelemetry/functions.sh @@ -9,35 +9,37 @@ fi function getAgent() { - if [ ! -f "${BASE_DIR}/lib/opentelemetry-javaagent.jar" ] - then - mkdir -p "${BASE_DIR}/lib" - wget --output-document="${AGENT_JAR}" \ - https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar - fi + if [ ! -f "${BASE_DIR}/lib/opentelemetry-javaagent.jar" ] + then + mkdir -p "${BASE_DIR}/lib" + wget --output-document="${AGENT_JAR}" \ + https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar + fi } function startJaeger { - if [ ! -d "${BASE_DIR}/jaeger-1.24.0-linux-amd64" ] ; then - cd "${BASE_DIR}" - wget https://github.com/jaegertracing/jaeger/releases/download/v1.24.0/jaeger-1.24.0-linux-amd64.tar.gz - tar -xvf jaeger-1.24.0-linux-amd64.tar.gz - rm jaeger-1.24.0-linux-amd64.tar.gz - fi - - cd "${BASE_DIR}/jaeger-1.24.0-linux-amd64" - "${BASE_DIR}/jaeger-1.24.0-linux-amd64/jaeger-all-in-one" &> "${BASE_DIR}/jaeger-1.24.0-linux-amd64/jaeger.log" & - cd "${BASE_DIR}" + if [ ! -d "${BASE_DIR}/jaeger-1.24.0-linux-amd64" ] ; then + cd "${BASE_DIR}" + wget https://github.com/jaegertracing/jaeger/releases/download/v1.24.0/jaeger-1.24.0-linux-amd64.tar.gz + tar -xvf jaeger-1.24.0-linux-amd64.tar.gz + rm jaeger-1.24.0-linux-amd64.tar.gz + fi + + cd "${BASE_DIR}/jaeger-1.24.0-linux-amd64" + "${BASE_DIR}/jaeger-1.24.0-linux-amd64/jaeger-all-in-one" &> "${BASE_DIR}/jaeger-1.24.0-linux-amd64/jaeger.log" & + cd "${BASE_DIR}" } function cleanup { - [ -f "${BASE_DIR}/hotspot.log" ] && mv "${BASE_DIR}/hotspot.log" "${RESULTS_DIR}/hotspot-${i}-$RECURSION_DEPTH-${k}.log" - echo >> "${BASE_DIR}/OpenTelemetry.log" - echo >> "${BASE_DIR}/OpenTelemetry.log" - sync - sleep "${SLEEP_TIME}" + [ -f "${BASE_DIR}/hotspot.log" ] && mv "${BASE_DIR}/hotspot.log" "${RESULTS_DIR}/hotspot-${i}-$RECURSION_DEPTH-${k}.log" + echo >> "${BASE_DIR}/OpenTelemetry.log" + echo >> "${BASE_DIR}/OpenTelemetry.log" + sync + sleep "${SLEEP_TIME}" } +# experiment setups + function runNoInstrumentation { # No instrumentation info " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" @@ -48,7 +50,7 @@ function runNoInstrumentation { --method-time "${METHOD_TIME}" \ --total-threads "${THREADS}" \ --recursion-depth "${RECURSION_DEPTH}" \ - ${MORE_PARAMS} #&> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" } function runOpenTelemetryNoLogging { @@ -58,11 +60,11 @@ function runOpenTelemetryNoLogging { echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >> "${BASE_DIR}/OpenTelemetry.log" export BENCHMARK_OPTS="${JAVA_ARGS_OPENTELEMETRY_LOGGING_DEACTIVATED}" "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv" \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth ${RECURSION_DEPTH} \ - ${MORE_PARAMS} #&> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" } function runOpenTelemetryLogging { @@ -71,16 +73,16 @@ function runOpenTelemetryLogging { info " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" echo " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" >> "${BASE_DIR}/OpenTelemetry.log" export BENCHMARK_OPTS="${JAVA_ARGS_OPENTELEMETRY_LOGGING}" - "${MOOBENCH_BIN}" --output-filename ${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth $RECURSION_DEPTH \ - ${MORE_PARAMS} #&> ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt + "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-${RECURSION_DEPTH}-${k}.csv" \ + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" if [ ! "$DEBUG" = true ] then - echo "DEBUG is $DEBUG, deleting opentelemetry logging file" - rm ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt + echo "DEBUG is $DEBUG, deleting opentelemetry logging file" + rm ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt fi } @@ -88,51 +90,51 @@ function runOpenTelemetryZipkin { # OpenTelemetry Instrumentation Zipkin k=`expr ${k} + 1` startZipkin - info " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} - echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >> "${BASE_DIR}/OpenTelemetry.log" + info " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" + echo " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" >> "${BASE_DIR}/OpenTelemetry.log" export BENCHMARK_OPTS="${JAVA_ARGS_OPENTELEMETRY_ZIPKIN}" - "${MOOBENCH_BIN}" --output-filename ${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth $RECURSION_DEPTH \ - ${MORE_PARAMS} #&> ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt + "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-${RECURSION_DEPTH}-${k}.csv" \ + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" stopBackgroundProcess - sleep $SLEEP_TIME + sleep "${SLEEP_TIME}" } function runOpenTelemetryJaeger { - # OpenTelemetry Instrumentation Jaeger - k=`expr ${k} + 1` - startJaeger - info " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} - echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >> "${BASE_DIR}/OpenTelemetry.log" - export BENCHMARK_OPTS="${JAVA_ARGS_OPENTELEMETRY_JAEGER}" - "${MOOBENCH_BIN}" --output-filename ${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth $RECURSION_DEPTH \ - ${MORE_PARAMS} #&> ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt - stopBackgroundProcess - sleep $SLEEP_TIME + # OpenTelemetry Instrumentation Jaeger + k=`expr ${k} + 1` + startJaeger + info " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" + echo " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" >> "${BASE_DIR}/OpenTelemetry.log" + export BENCHMARK_OPTS="${JAVA_ARGS_OPENTELEMETRY_JAEGER}" + "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-${RECURSION_DEPTH}-${k}.csv" \ + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" + stopBackgroundProcess + sleep "${SLEEP_TIME}" } function runOpenTelemetryPrometheus { - # OpenTelemetry Instrumentation Prometheus - k=`expr ${k} + 1` - startPrometheus - info " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} - echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >> "${BASE_DIR}/OpenTelemetry.log" - export BENCHMARK_OPTS="${JAVA_ARGS_OPENTELEMETRY_PROMETHEUS}" - "${MOOBENCH_BIN}" --output-filename ${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth $RECURSION_DEPTH \ - ${MORE_PARAMS} #&> ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt - stopBackgroundProcess - sleep $SLEEP_TIME + # OpenTelemetry Instrumentation Prometheus + k=`expr ${k} + 1` + startPrometheus + info " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" + echo " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" >> "${BASE_DIR}/OpenTelemetry.log" + export BENCHMARK_OPTS="${JAVA_ARGS_OPENTELEMETRY_PROMETHEUS}" + "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-${RECURSION_DEPTH}-${k}.csv" \ + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" + stopBackgroundProcess + sleep $SLEEP_TIME } # end diff --git a/frameworks/common-functions.sh b/frameworks/common-functions.sh index 1b0509f26d688992a01c498bea98d13b1e5e8111..a021fe8c1e7f5bd90127e347ab71a399b06d9293 100755 --- a/frameworks/common-functions.sh +++ b/frameworks/common-functions.sh @@ -43,24 +43,13 @@ function createRLabels() { ## Generate Results file function runStatistics() { -cat << EOF -results_fn="${RAWFN}" -out_yaml_fn="${RESULTS_DIR}/results.yaml" -configs.loop=${NUM_OF_LOOPS} -configs.recursion=${RECURSION_DEPTH} -configs.labels=c($LABELS) -configs.tool_id="$RESULTS_DIR}" -results.count=${TOTAL_NUM_OF_CALLS} -results.skip=${TOTAL_NUM_OF_CALLS}/2 -source("${RSCRIPT_PATH}") -EOF R --vanilla --silent << EOF results_fn="${RAWFN}" out_yaml_fn="${RESULTS_DIR}/results.yaml" configs.loop=${NUM_OF_LOOPS} configs.recursion=${RECURSION_DEPTH} configs.labels=c($LABELS) -configs.tool_id="$RESULTS_DIR}" +configs.framework_name="${FRAMEWORK_NAME}" results.count=${TOTAL_NUM_OF_CALLS} results.skip=${TOTAL_NUM_OF_CALLS}/2 source("${RSCRIPT_PATH}") @@ -228,7 +217,7 @@ if [ -z $SLEEP_TIME ]; then SLEEP_TIME=30 ## 30 fi if [ -z $NUM_OF_LOOPS ]; then - NUM_OF_LOOPS=1 #0 ## 10 + NUM_OF_LOOPS=10 ## 10 fi if [ -z $THREADS ]; then THREADS=1 ## 1 diff --git a/frameworks/inspectIT/benchmark.sh b/frameworks/inspectIT/benchmark.sh index 5aa00027b2154b3be0316e38b1eadcf05d19b737..385766f3bec95058bbf6ac2cfdc93a8875ab5cbf 100755 --- a/frameworks/inspectIT/benchmark.sh +++ b/frameworks/inspectIT/benchmark.sh @@ -56,19 +56,12 @@ info "----------------------------------" # load agent getAgent -checkFile MooBench "${BASE_DIR}/MooBench.jar" +checkExecutable MooBench "${MOOBENCH_BIN}" checkFile log "${BASE_DIR}/inspectIT.log" clean checkDirectory results-directory "${RESULTS_DIR}" recreate checkExecutable java "${JAVA_BIN}" checkFile R-script "${RSCRIPT_PATH}" -# -# Run benchmark -# - -info "----------------------------------" -info "Running benchmark..." -info "----------------------------------" TIME=`expr ${METHOD_TIME} \* ${TOTAL_NUM_OF_CALLS} / 1000000000 \* 4 \* ${RECURSION_DEPTH} \* ${NUM_OF_LOOPS} + ${SLEEP_TIME} \* 4 \* ${NUM_OF_LOOPS} \* ${RECURSION_DEPTH} + 50 \* ${TOTAL_NUM_OF_CALLS} / 1000000000 \* 4 \* ${RECURSION_DEPTH} \* ${NUM_OF_LOOPS} ` info "Experiment will take circa ${TIME} seconds." @@ -76,7 +69,6 @@ info "Experiment will take circa ${TIME} seconds." JAVA_ARGS="-server" JAVA_ARGS="${JAVA_ARGS} -Xms1G -Xmx2G" JAVA_ARGS="${JAVA_ARGS} -verbose:gc " -JAR="-jar ${BASE_DIR}/MooBench.jar --application moobench.application.MonitoredClassSimple" JAVA_ARGS_NOINSTR="${JAVA_ARGS}" JAVA_ARGS_LTW="${JAVA_ARGS} -javaagent:${BASE_DIR}/agent/inspectit-ocelot-agent-1.11.1.jar -Djava.util.logging.config.file=${BASE_DIR}/config/logging.properties" @@ -89,7 +81,13 @@ info "RESULTS_DIR: ${RESULTS_DIR}" info "RAWFN: $RAWFN" writeConfiguration -info "Ok" +# +# Run benchmark +# + +info "----------------------------------" +info "Running benchmark..." +info "----------------------------------" ## Execute Benchmark for ((i=1;i<=${NUM_OF_LOOPS};i+=1)); do @@ -121,7 +119,7 @@ mv "${BASE_DIR}/inspectIT.log" "${RESULTS_DIR}/inspectIT.log" # Create R labels LABELS=$(createRLabels) -runR +runStatistics ## Clean up raw results zip -jqr "${RESULTS_DIR}/results.zip" ${RAWFN}* diff --git a/frameworks/inspectIT/config.rc b/frameworks/inspectIT/config.rc index 8453f04142ab6f209b7637948d82475f8f9f367e..887ca4d0d8c98718f820c60ca89c600870895f7b 100644 --- a/frameworks/inspectIT/config.rc +++ b/frameworks/inspectIT/config.rc @@ -1,10 +1,12 @@ -# inspectIT - -RSCRIPT_PATH="${BASE_DIR}/../statistics.r" +# inspectIT configurations JAVA_BIN="/usr/bin/java" +RSCRIPT_PATH="${BASE_DIR}/../statistics.r" + #MORE_PARAMS="--quickstart" MORE_PARAMS="${MORE_PARAMS}" +MOOBENCH_BIN="${BASE_DIR}/../../benchmark/bin/benchmark" + diff --git a/frameworks/inspectIT/functions.sh b/frameworks/inspectIT/functions.sh index 292f5e993a5c85b2ab8a7c10a52e7693418ec3ec..c65b22737fb55c1719e693cc7040935019b5953b 100644 --- a/frameworks/inspectIT/functions.sh +++ b/frameworks/inspectIT/functions.sh @@ -7,6 +7,7 @@ then exit 1 fi + function getAgent() { if [ ! -d "${BASE_DIR}/agent" ] ; then mkdir "${BASE_DIR}/agent" @@ -16,99 +17,103 @@ function getAgent() { fi } +function cleanup { + [ -f "${BASE_DIR}/hotspot.log" ] && mv "${BASE_DIR}/hotspot.log" "${RESULTS_DIR}/hotspot-${i}-${j}-${k}.log" + echo >> "${BASE_DIR}/inspectIT.log" + echo >> "${BASE_DIR}/inspectIT.log" + sync + sleep "${SLEEP_TIME}" +} + +function getSum { + awk '{sum += $1; square += $1^2} END {print "Average: "sum/NR" Standard Deviation: "sqrt(square / NR - (sum/NR)^2)" Count: "NR}' +} + + +# experiment setups + function runNoInstrumentation { # No instrumentation - info " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} - echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >>${BASE_DIR}/inspectIT.log - ${JAVA_BIN} ${JAVA_ARGS_NOINSTR} ${JAR} \ - --output-filename ${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth ${RECURSION_DEPTH} \ - ${MORE_PARAMS} &> ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt + info " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" + echo " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" >> "${BASE_DIR}/inspectIT.log" + export BENCHMARK_OPTS="${JAVA_ARGS_NOINSTR}" + "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv" \ + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" } function runInspectITDeactivated { k=`expr ${k} + 1` info " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} - echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >>${BASE_DIR}/inspectIT.log - sleep $SLEEP_TIME - ${JAVA_BIN} ${JAVA_ARGS_INSPECTIT_DEACTIVATED} ${JAR} \ - --output-filename ${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth ${RECURSION_DEPTH} \ + echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >> "${BASE_DIR}/inspectIT.log" + sleep "${SLEEP_TIME}" + export BENCHMARK_OPTS="${JAVA_ARGS_INSPECTIT_DEACTIVATED}" + "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv" \ + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ --force-terminate \ - ${MORE_PARAMS} &> ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt - sleep $SLEEP_TIME + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" + sleep "${SLEEP_TIME}" } function runInspectITNullWriter { k=`expr ${k} + 1` info " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >>${BASE_DIR}/inspectIT.log - sleep $SLEEP_TIME - ${JAVA_BIN} ${JAVA_ARGS_INSPECTIT_NULLWRITER} ${JAR} \ - --output-filename ${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth ${RECURSION_DEPTH} \ + sleep "${SLEEP_TIME}" + export BENCHMARK_OPTS="${JAVA_ARGS_INSPECTIT_NULLWRITER}" + "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-${RECURSION_DEPTH}-${k}.csv" \ + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ --force-terminate \ - ${MORE_PARAMS} &> ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt - sleep $SLEEP_TIME + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" + sleep "${SLEEP_TIME}" } function runInspectITZipkin { # InspectIT (minimal) k=`expr ${k} + 1` - info " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} - echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >>${BASE_DIR}/inspectIT.log + info " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" + echo " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" >> "${BASE_DIR}/inspectIT.log" startZipkin - sleep $SLEEP_TIME - ${JAVA_BIN} ${JAVA_ARGS_INSPECTIT_ZIPKIN} ${JAR} \ - --output-filename ${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth ${RECURSION_DEPTH} \ + sleep "${SLEEP_TIME}" + export BENCHMARK_OPTS="${JAVA_ARGS_INSPECTIT_ZIPKIN}" + "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-${RECURSION_DEPTH}-${k}.csv" \ + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ --force-terminate \ - ${MORE_PARAMS} &> ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" stopBackgroundProcess - sleep $SLEEP_TIME + sleep "${SLEEP_TIME}" } function runInspectITPrometheus { # InspectIT (minimal) k=`expr ${k} + 1` - info " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} - echo " # ${i}.$RECURSION_DEPTH.${k} "${TITLE[$k]} >>${BASE_DIR}/inspectIT.log + info " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" + echo " # ${i}.$RECURSION_DEPTH.${k} ${TITLE[$k]}" >> "${BASE_DIR}/inspectIT.log" startPrometheus - sleep $SLEEP_TIME - ${JAVA_BIN} ${JAVA_ARGS_INSPECTIT_PROMETHEUS} ${JAR} \ - --output-filename ${RAWFN}-${i}-$RECURSION_DEPTH-${k}.csv \ - --total-calls ${TOTAL_NUM_OF_CALLS} \ - --method-time ${METHOD_TIME} \ - --total-threads ${THREADS} \ - --recursion-depth ${RECURSION_DEPTH} \ + sleep "${SLEEP_TIME}" + export BENCHMARK_OPTS="${JAVA_ARGS_INSPECTIT_PROMETHEUS}" + "${MOOBENCH_BIN}" --output-filename "${RAWFN}-${i}-${RECURSION_DEPTH}-${k}.csv" \ + --total-calls "${TOTAL_NUM_OF_CALLS}" \ + --method-time "${METHOD_TIME}" \ + --total-threads "${THREADS}" \ + --recursion-depth "${RECURSION_DEPTH}" \ --force-terminate \ - ${MORE_PARAMS} &> ${RESULTS_DIR}/output_"$i"_"$RECURSION_DEPTH"_$k.txt + ${MORE_PARAMS} &> "${RESULTS_DIR}/output_${i}_${RECURSION_DEPTH}_${k}.txt" stopBackgroundProcess sleep $SLEEP_TIME } -function cleanup { - [ -f "${BASE_DIR}/hotspot.log" ] && mv "${BASE_DIR}/hotspot.log" "${RESULTS_DIR}/hotspot-${i}-${j}-${k}.log" - echo >> "${BASE_DIR}/inspectIT.log" - echo >> "${BASE_DIR}/inspectIT.log" - sync - sleep "${SLEEP_TIME}" -} - -function getSum { - awk '{sum += $1; square += $1^2} END {print "Average: "sum/NR" Standard Deviation: "sqrt(square / NR - (sum/NR)^2)" Count: "NR}' -} - +# end diff --git a/frameworks/statistics.r b/frameworks/statistics.r index 701f50b64385ff5b5718ebdee05117a9b115cf90..d97c6435fba693f2b26da47ee5f6d8bcbf2f733b 100644 --- a/frameworks/statistics.r +++ b/frameworks/statistics.r @@ -17,7 +17,7 @@ #configs.loop=10 #configs.recursion=c(10) #configs.labels=c("No Probe","Inactive Probe","Collecting Data","Writing Data (ASCII)", "Writing Data (Bin)") -#configs.tool_id="kieker-java" +#configs.framework_name="kieker-java" #results.count=2000000 #results.skip=1000000 @@ -90,7 +90,7 @@ print(resultstext) currentTime <- as.numeric(Sys.time()) -write(paste(configs.tool_id ":"), file=out_yaml_fn,append=FALSE) +write(paste(configs.framework_name, ":"), file=out_yaml_fn,append=FALSE) write(paste("- timestamp:", currentTime), file=out_yaml_fn, append=TRUE) for (writer_idx in (1:(numberOfWriters))) { write(paste(" ", configs.labels[writer_idx], ": [", diff --git a/run-benchmark.sh b/run-benchmark.sh deleted file mode 100755 index eb33a2ff64e985a0b93cee6d2a0b7072525009b7..0000000000000000000000000000000000000000 --- a/run-benchmark.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -# -# Jenkinsfile benchmark runner -# - -BASE_DIR=$(cd "$(dirname "$0")"; pwd) - -# load configuration and common functions -if [ -f "${BASE_DIR}/config" ] ; then - . "${BASE_DIR}/config" -else - echo "Missing configuration: ${BASE_DIR}/config" - exit 1 -fi -if [ -f "${BASE_DIR}/../common-functions.sh" ] ; then - . "${BASE_DIR}/../common-functions.sh" -else - echo "Missing configuration: ${BASE_DIR}/../common-functions.sh" - exit 1 -fi - -# -# setup -# - -export RESULT_FILE="${RESULTS_DIR}/results-text.csv" -COLLECTED_DATA_FILE="${BASE_DIR}/results.csv" -BENCHMARK="${BASE_DIR}/benchmark.sh" - -# -# Setup -# - -cd ${BASE_DIR} - -# install benchmark -tar -xvpf ${BASE_DIR}/../../benchmark/build/distributions/benchmark.tar -# copy receiver -tar -xvpf ${BASE_DIR}/../../tools/receiver/build/distributions/receiver.tar -# copy result compiler -tar -xvpf ${BASE_DIR}/../../tools/compile-results/build/distributions/compile-results.tar - -# Create benchmark results -mkdir -p ${RESULTS_DIR} - -rm -f ${COLLECTED_DATA_FILE} - -## running the benchmark -${BENCHMARK} # > /dev/null 2>&1 - -# end diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000000000000000000000000000000000000..8c6c49414aec9901f4ac5df3792ef188987682cc --- /dev/null +++ b/setup.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# +# Kieker moobench setup script +# +# Usage: setup.sh + +# configure base dir +BASE_DIR=$(cd "$(dirname "$0")"; pwd) + +# +# source functionality +# + +if [ ! -d "${BASE_DIR}" ] ; then + echo "Base directory ${BASE_DIR} does not exist." + exit 1 +fi + +# load configuration and common functions +if [ -f "${BASE_DIR}/config.rc" ] ; then + source "${BASE_DIR}/config.rc" +else + echo "Missing configuration: ${BASE_DIR}/config.rc" + exit 1 +fi + +if [ -f "${BASE_DIR}/frameworks/common-functions.sh" ] ; then + source "${BASE_DIR}/frameworks/common-functions.sh" +else + echo "Missing library: ${BASE_DIR}/frameworks/common-functions.sh" + exit 1 +fi + +cd "${BASE_DIR}" + +./gradlew build + +checkFile moobench "${MOOBENCH_ARCHIVE}" +tar -xpf "${MOOBENCH_ARCHIVE}" +MOOBENCH_BIN="${BASE_DIR}/benchmark/bin/benchmark" + +# end