diff --git a/README.md b/README.md index e6cc5b1dcb3519afcf61d77376718a0212145a63..5ee15bb7b706c875b389686adefe756bfac13291 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,44 @@ -The MooBench Monitoring Overhead Micro-Benchmark ------------------------------------------------------------------------- +# The MooBench Monitoring Overhead Micro-Benchmark Website: http://kieker-monitoring.net/MooBench -Note: Please note that we are currently reorganizing the project structure. -Thus, the documentation might be outdated. +The MooBench micro-benchmarks can be used to quantify the performance overhead caused by monitoring framework components. -The MooBench micro-benchmarks can be used to quantify the performance -overhead caused by monitoring framework components. - -Currenly (directly) supported monitoring frameworks are: +Currenly (fully) supported monitoring frameworks are: * Kieker (http://kieker-monitoring.net) * OpenTelemetry (https://opentelemetry.io/) -* SPASS-meter (https://github.com/SSEHUB/spassMeter.git) - -The gradle buildfile is provided to prepare the benchmark. To build -the monitored application and copy it to the framework you want to benchmark, -just execute `./gradlew assemble` -All experiments are started with the provided "External Controller" -scripts. These scripts are available inside the respective bin/ -directory. Currently only shell (.sh) scripts are provided. These -scripts have been developed on Solaris environments. Thus, minor -adjustments might be required for common Linux operatong systems, -such as Ubuntu. Additionally, several Eclipse launch targets are -provided for debugging purposes. - -The default execution of the benchmark requires a 64Bit JVM! -However, this behavior can be changed in the respective .sh scripts. +## Benchmark Execution Initially, the following steps are required: -1. Make sure, that you've installed R (http://www.r-project.org/) to - generate the results. +1. Make sure, that you've installed R (http://www.r-project.org/) to generate the results (Ubuntu: `sudo apt install r-base`) and awk to install intermediate results (Ubuntu: `sudo apt install gawk`). 2. Compile the application by calling `./gradlew assemble`. -Execution of the micro-benchmark: -All benchmarks are started with calls of .sh scripts in the bin folder. -The top of the files include some configuration parameters, such as -* SLEEPTIME between executions (default 30 seconds) -* NUM_LOOPS number of repetitions (default 10) -* THREADS concurrent benchmarking threads (default 1) -* MAXRECURSIONDEPTH recursion up to this depth (default 10) -* TOTALCALLS the duration of the benchmark (deafult 2,000,000 calls) -* METHODTIME the time per monitored call (default 0 ns or 500 us) +All experiments are started with the provided "External Controller" scripts. The following scripts are available +* for Kieker: In `frameworks/Kieker/scripts/benchmark.sh` for regular execution and `frameworks/Kieker/scripts/runExponentialSizes.sh` for execution of different call tree depth sizes +* for OpenTelemetry: `frameworks/opentelemetry/benchmark.sh` for regular execution and `frameworks/opentelemetry/runExponentialSizes.sh` for execution of different call tree depth sizes + +All scripts have been tested on Ubuntu and Raspbian. -Furthermore some JVM arguments can be adjusted: -* JAVAARGS JVM Arguments (e.g., available memory) +The execution may be parameterized by the following environment variables: +* SLEEP_TIME between executions (default 30 seconds) +* NUM_OF_LOOPS number of repetitions (default 10) +* THREADS concurrent benchmarking threads (default 1) +* RECURSION_DEPTH recursion up to this depth (default 10) +* TOTAL_NUM_OF_CALLS the duration of the benchmark (deafult 2,000,000 calls) +* METHOD_TIME the time per monitored call (default 0 ns or 500 us) -Typical call (using Solaris): -$ nohup ./benchmark.sh & sleep 1;tail +0cf nohup.out +If they are unset, the values are set via `frameworks/common-function.sh`. +Typical call (using Ubuntu): +``` +export SLEEP_TIME=1 +./gradlew assemble +cd frameworks/opentelemetry/ +./benchmark.sh +``` -Analyzing the data: -=================== +## Data Analysis In the folder /bin/r are some R scripts provided to generate graphs to visualize the results. In the top the files, one can configure the required paths and the configuration used to analyze the data. diff --git a/frameworks/Kieker/scripts/benchmark.sh b/frameworks/Kieker/scripts/benchmark.sh index f8633e42e6ed2a8c8aad31f6269cc11922682d8f..be20816f47ba768c0e738e18b9ade8abf7e6b50e 100755 --- a/frameworks/Kieker/scripts/benchmark.sh +++ b/frameworks/Kieker/scripts/benchmark.sh @@ -18,6 +18,8 @@ fi source ../../common-functions.sh +getKiekerAgent + if [ -f "${BASE_DIR}/common-functions" ] ; then . ${BASE_DIR}/common-functions else @@ -104,15 +106,7 @@ RECEIVER[5]="${BASE_DIR}/collector-2.0/bin/collector -p 2345" RECEIVER[5]="${BASE_DIR}/receiver/bin/receiver 2345" # Create R labels -LABELS="" -for I in "${TITLE[@]}" ; do - title="$I" - if [ "$LABELS" == "" ] ; then - LABELS="\"$title\"" - else - LABELS="${LABELS}, \"$title\"" - fi -done +LABELS=$(createRLabels) ## Write configuration uname -a >${RESULTS_DIR}/configuration.txt diff --git a/frameworks/Kieker/scripts/run-benchmark.sh b/frameworks/Kieker/scripts/run-benchmark.sh index 4a2b5854afbccb50180c448ae3179f7d71858d1d..b80da7c8f2ae923edcbbd11ed758c27c3a1d94f3 100755 --- a/frameworks/Kieker/scripts/run-benchmark.sh +++ b/frameworks/Kieker/scripts/run-benchmark.sh @@ -28,10 +28,6 @@ cd ${BASE_DIR} ## setup # install benchmark tar -xvpf ${BASE_DIR}/../../../benchmark/build/distributions/benchmark.tar -# get agent -export VERSION_PATH=`curl "https://oss.sonatype.org/service/local/repositories/snapshots/content/net/kieker-monitoring/kieker/" | grep '<resourceURI>' | sed 's/ *<resourceURI>//g' | sed 's/<\/resourceURI>//g' | grep '/$'` -export AGENT_PATH=`curl "${VERSION_PATH}" | grep 'aspectj.jar</resourceURI' | sort | sed 's/ *<resourceURI>//g' | sed 's/<\/resourceURI>//g' | tail -1` -curl "${AGENT_PATH}" > "${AGENT}" # copy receiver tar -xvpf ${BASE_DIR}/../../../tools/receiver/build/distributions/receiver.tar # copy result compiler diff --git a/frameworks/common-functions.sh b/frameworks/common-functions.sh index 11ae83f33145e4e94d7a0d09a4386bd886278b70..5505acc8b1299247ea9731e39ab4d6d37a6b05ea 100644 --- a/frameworks/common-functions.sh +++ b/frameworks/common-functions.sh @@ -12,6 +12,31 @@ function cleanup-results() { [ -f ${DATA_DIR}/nohup.out ] && > ${DATA_DIR}/nohup.out } +function getKiekerAgent() { + echo "Checking whether Kieker is present in $AGENT" + if [ ! -f $AGENT ] + then + # get agent + export VERSION_PATH=`curl "https://oss.sonatype.org/service/local/repositories/snapshots/content/net/kieker-monitoring/kieker/" | grep '<resourceURI>' | sed 's/ *<resourceURI>//g' | sed 's/<\/resourceURI>//g' | grep '/$'` + export AGENT_PATH=`curl "${VERSION_PATH}" | grep 'aspectj.jar</resourceURI' | sort | sed 's/ *<resourceURI>//g' | sed 's/<\/resourceURI>//g' | tail -1` + curl "${AGENT_PATH}" > "${AGENT}" + fi +} + +function createRLabels() { + # Create R labels + LABELS="" + for I in "${TITLE[@]}" ; do + title="$I" + if [ "$LABELS" == "" ] ; then + LABELS="\"$title\"" + else + LABELS="${LABELS}, \"$title\"" + fi + done + echo $LABELS +} + # Initialize all unset parameters if [ -z $SLEEP_TIME ]; then SLEEP_TIME=30 ## 30 diff --git a/frameworks/opentelemetry/benchmark.sh b/frameworks/opentelemetry/benchmark.sh index d26058d7aefbbdac50e33b61951f5a55a69f1ba0..da30cc2c788f21b87179fd6adf5948dd8fc428ec 100755 --- a/frameworks/opentelemetry/benchmark.sh +++ b/frameworks/opentelemetry/benchmark.sh @@ -170,6 +170,15 @@ function printIntermediaryResults { fi } +function downloadOpentelemetry() { + if [ ! -f ${BASEDIR}lib/opentelemetry-javaagent-all.jar ] + then + mkdir -p ${BASEDIR}lib + wget --output-document=${BASEDIR}lib/opentelemetry-javaagent-all.jar \ + https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent-all.jar + fi +} + JAVABIN="" @@ -200,9 +209,6 @@ JAVAARGS="-server" JAVAARGS="${JAVAARGS} " JAVAARGS="${JAVAARGS} -Xms1G -Xmx2G" JAVAARGS="${JAVAARGS} -verbose:gc -XX:+PrintCompilation" -#JAVAARGS="${JAVAARGS} -XX:+PrintInlining" -#JAVAARGS="${JAVAARGS} -XX:+UnlockDiagnosticVMOptions -XX:+LogCompilation" -#JAVAARGS="${JAVAARGS} -Djava.compiler=NONE" JAR="-jar MooBench.jar" if [ ! -f "MooBench.jar" ] @@ -211,12 +217,7 @@ then exit 1 fi -if [ ! -f ${BASEDIR}lib/opentelemetry-javaagent-all.jar ] -then - mkdir -p ${BASEDIR}lib - wget --output-document=${BASEDIR}lib/opentelemetry-javaagent-all.jar \ - https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent-all.jar -fi +downloadOpentelemetry JAVAARGS_NOINSTR="${JAVAARGS}" JAVAARGS_OPENTELEMETRY_BASIC="${JAVAARGS} -javaagent:${BASEDIR}lib/opentelemetry-javaagent-all.jar -Dotel.resource.attributes=service.name=moobench -Dotel.instrumentation.methods.include=moobench.application.MonitoredClassSimple[monitoredMethod];moobench.application.MonitoredClassThreaded[monitoredMethod]"