Merge branch 'enhance-error-handling' into 'master'

Throw exception in order to set the execution states of an execution correctly

See merge request !175

theodolite/src/main/kotlin/theodolite/evaluation/AnalysisExecutor.kt

@@ -2,6 +2,7 @@ package theodolite.evaluation
 
 import mu.KotlinLogging
 import theodolite.benchmark.BenchmarkExecution
+import theodolite.util.EvaluationFailedException
 import theodolite.util.IOHandler
 import theodolite.util.LoadDimension
 import theodolite.util.Resource
@@ -37,7 +38,7 @@ class AnalysisExecutor(
      *  @return true if the experiment succeeded.
      */
     fun analyze(load: LoadDimension, res: Resource, executionIntervals: List<Pair<Instant, Instant>>): Boolean {
-        var result = false
+        var result: Boolean
         var repetitionCounter = 1
 
         try {
@@ -71,8 +72,7 @@ class AnalysisExecutor(
             result = sloChecker.evaluate(prometheusData)
 
         } catch (e: Exception) {
-            // TODO(throw exception in order to make it possible to mark an experiment as unsuccessfully)
-            logger.error { "Evaluation failed for resource '${res.get()}' and load '${load.get()}'. Error: $e" }
+            throw EvaluationFailedException("Evaluation failed for resource '${res.get()}' and load '${load.get()} ", e)
         }
         return result
     }

theodolite/src/main/kotlin/theodolite/evaluation/MetricFetcher.kt

@@ -53,8 +53,7 @@ class MetricFetcher(private val prometheusURL: String, private val offset: Durat
             } else {
                 val values = parseValues(response)
                 if (values.data?.result.isNullOrEmpty()) {
-                    logger.error { "Empty query result: $values between $start and $end for query $query." }
-                    throw NoSuchFieldException()
+                    throw NoSuchFieldException("Empty query result: $values between $start and $end for query $query.")
                 }
                 return parseValues(response)
             }

theodolite/src/main/kotlin/theodolite/execution/BenchmarkExecutorImpl.kt

@@ -5,10 +5,7 @@ import mu.KotlinLogging
 import theodolite.benchmark.Benchmark
 import theodolite.benchmark.BenchmarkExecution
 import theodolite.evaluation.AnalysisExecutor
-import theodolite.util.ConfigurationOverride
-import theodolite.util.LoadDimension
-import theodolite.util.Resource
-import theodolite.util.Results
+import theodolite.util.*
 import java.time.Duration
 import java.time.Instant
 
@@ -61,6 +58,11 @@ class BenchmarkExecutorImpl(
                 )
             this.results.setResult(Pair(load, res), result)
         }
+
+        if(!this.run.get()) {
+            throw ExecutionFailedException("The execution was interrupted")
+        }
+
         return result
     }
 
@@ -73,21 +75,19 @@ class BenchmarkExecutorImpl(
             this.afterTeardownDelay
         )
         val from = Instant.now()
-        // TODO(restructure try catch in order to throw exceptions if there are significant problems by running a experiment)
+
         try {
             benchmarkDeployment.setup()
             this.waitAndLog()
         } catch (e: Exception) {
-            logger.error { "Error while setup experiment." }
-            logger.error { "Error is: $e" }
             this.run.set(false)
+            throw ExecutionFailedException("Error during setup the experiment", e)
         }
         val to = Instant.now()
         try {
             benchmarkDeployment.teardown()
         } catch (e: Exception) {
-            logger.warn { "Error while tearing down the benchmark deployment." }
-            logger.debug { "Teardown failed, caused by: $e" }
+            throw ExecutionFailedException("Error during teardown the experiment", e)
         }
         return Pair(from, to)
     }

theodolite/src/main/kotlin/theodolite/execution/Shutdown.kt

@@ -34,16 +34,15 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b
                     afterTeardownDelay = 5L
                 )
             deployment.teardown()
+            logger.info {
+                "Finished teardown of all benchmark resources."
+            }
         } catch (e: Exception) {
-            // TODO(throw exception in order to make it possible to mark an experiment as unsuccessfully)
             logger.warn {
                 "Could not delete all specified resources from Kubernetes. " +
                         "This could be the case, if not all resources are deployed and running."
             }
 
         }
-        logger.info {
-            "Finished teardown of all benchmark resources."
-        }
     }
 }

theodolite/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt

@@ -123,15 +123,18 @@ class TheodoliteExecutor(
 
         val config = buildConfig()
         // execute benchmarks for each load
-        for (load in config.loads) {
-            if (executor.run.get()) {
-                config.compositeStrategy.findSuitableResource(load, config.resources)
+        try {
+            for (load in config.loads) {
+                if (executor.run.get()) {
+                    config.compositeStrategy.findSuitableResource(load, config.resources)
+                }
             }
+        } finally {
+            ioHandler.writeToJSONFile(
+                config.compositeStrategy.benchmarkExecutor.results,
+                "${resultsFolder}exp${this.config.executionId}-result"
+            )
         }
-        ioHandler.writeToJSONFile(
-            config.compositeStrategy.benchmarkExecutor.results,
-            "${resultsFolder}exp${this.config.executionId}-result"
-        )
     }
 
     private fun getAndIncrementExecutionID(fileURL: String): Int {

theodolite/src/main/kotlin/theodolite/execution/TheodoliteStandalone.kt

@@ -3,6 +3,8 @@ package theodolite.execution
 import mu.KotlinLogging
 import theodolite.benchmark.BenchmarkExecution
 import theodolite.benchmark.KubernetesBenchmark
+import theodolite.util.EvaluationFailedException
+import theodolite.util.ExecutionFailedException
 import theodolite.util.YamlParser
 import kotlin.concurrent.thread
 import kotlin.system.exitProcess
@@ -49,8 +51,14 @@ class TheodoliteStandalone {
         val shutdown = thread(start = false) { Shutdown(benchmarkExecution, benchmark).run() }
         Runtime.getRuntime().addShutdownHook(shutdown)
 
-        val executor = TheodoliteExecutor(benchmarkExecution, benchmark)
-        executor.run()
+        try {
+            TheodoliteExecutor(benchmarkExecution, benchmark).run()
+        } catch (e: EvaluationFailedException) {
+            logger.error { "Evaluation failed with error: ${e.message}" }
+        }catch (e: ExecutionFailedException) {
+            logger.error { "Execution failed with error: ${e.message}" }
+        }
+
         logger.info { "Theodolite finished" }
         Runtime.getRuntime().removeShutdownHook(shutdown)
         exitProcess(0)

theodolite/src/main/kotlin/theodolite/execution/operator/ClusterSetup.kt

@@ -54,11 +54,8 @@ class ClusterSetup(
                     benchmark.spec.name = benchmark.metadata.name
                     Shutdown(execution.spec, benchmark.spec).start()
                 } else {
-                    logger.error {
-                        "Execution with state ${States.RUNNING.value} was found, but no corresponding benchmark. " +
-                                "Could not initialize cluster."
-                    }
-                    throw IllegalStateException("Cluster state is invalid, required Benchmark for running execution not found.")
+                    throw IllegalStateException("Execution with state ${States.RUNNING.value} was found, but no corresponding benchmark. " +
+                            "Could not initialize cluster.")
                 }
             }
     }

theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt

@@ -8,6 +8,7 @@ import theodolite.benchmark.KubernetesBenchmark
 import theodolite.execution.TheodoliteExecutor
 import theodolite.model.crd.*
 import theodolite.patcher.ConfigOverrideModifier
+import theodolite.util.ExecutionFailedException
 import theodolite.util.ExecutionStateComparator
 import java.lang.Thread.sleep
 
@@ -93,6 +94,10 @@ class TheodoliteController(
                 States.RUNNING -> {
                     executionStateHandler.setExecutionState(execution.name, States.FINISHED)
                     logger.info { "Execution of ${execution.name} is finally stopped." }
+                    }
+                else -> {
+                    executionStateHandler.setExecutionState(execution.name, States.FAILURE)
+                    logger.warn { "Unexpected execution state, set state to ${States.FAILURE.value}" }
                 }
             }
         } catch (e: Exception) {

theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt

@@ -33,7 +33,7 @@ class CustomResourceWrapper(
             client.customResource(this.context)
                 .delete(client.configuration.namespace, this.getName())
         } catch (e: Exception) {
-            logger.warn { "Could not delete service monitor" }
+            logger.warn { "Could not delete custom resource" }
         }
     }
 

theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt

@@ -7,6 +7,7 @@ import io.fabric8.kubernetes.api.model.apps.Deployment
 import io.fabric8.kubernetes.client.NamespacedKubernetesClient
 import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext
 import mu.KotlinLogging
+import theodolite.util.DeploymentFailedException
 import theodolite.util.YamlParser
 
 private val logger = KotlinLogging.logger {}
@@ -120,10 +121,8 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
         try {
             resource = f(path)
         } catch (e: Exception) {
-            logger.warn { "You potentially misspelled the path: $path" }
-            logger.warn { e }
+            throw IllegalArgumentException("The Resource at path: $path could not be loaded", e)
         }
-
         if (resource == null) {
             throw IllegalArgumentException("The Resource at path: $path could not be loaded")
         }
@@ -147,10 +146,7 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
             "StatefulSet" -> loadStatefulSet(path)
             "Execution" -> loadExecution(path)
             "Benchmark" -> loadBenchmark(path)
-            else -> {
-                logger.error { "Error during loading of unspecified resource Kind" }
-                throw IllegalArgumentException("error while loading resource with kind: $kind")
-            }
+            else -> throw IllegalArgumentException("error while loading resource with kind: $kind")
         }
     }
 }

theodolite/src/main/kotlin/theodolite/k8s/TopicManager.kt

@@ -30,8 +30,7 @@ class TopicManager(private val kafkaConfig: Map<String, Any>) {
                 result = kafkaAdmin.createTopics(newTopics)
                 result.all().get() // wait for the future to be completed
             } catch (e: Exception) { // TopicExistsException
-                logger.warn(e) { "Error during topic creation." }
-                logger.debug { e } // TODO remove due to attached exception to warn log?
+                logger.warn { "Error during topic creation. Error is: ${e.message}" }
                 logger.info { "Remove existing topics." }
                 delete(newTopics.map { topic -> topic.name() }, kafkaAdmin)
                 logger.info { "Will retry the topic creation in ${RETRY_TIME / 1000} seconds." }
@@ -94,7 +93,7 @@ class TopicManager(private val kafkaConfig: Map<String, Any>) {
                     }"
                 }
             } catch (e: Exception) {
-                logger.error(e) { "Error while removing topics: $e" }
+                logger.error { "Error while removing topics: ${e.message}" }
                 logger.info { "Existing topics are: ${kafkaAdmin.listTopics().names().get()}." }
             }
 

theodolite/src/main/kotlin/theodolite/util/DeploymentFailedException.kt

 package theodolite.util
 
 
-class DeploymentFailedException(message: String) : Exception(message)
\ No newline at end of file
+open class DeploymentFailedException(message: String, e: Exception? = null) : TheodoliteException(message,e)
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/util/EvaluationFailedException.kt

+package theodolite.util
+
+class EvaluationFailedException(message: String, e: Exception? = null) : ExecutionFailedException(message,e) {
+}

theodolite/src/main/kotlin/theodolite/util/ExecutionFailedException.kt

+package theodolite.util
+
+open class ExecutionFailedException(message: String, e: Exception? = null) : TheodoliteException(message,e) {
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/util/InvalidPatcherConfigurationException.kt

 package theodolite.util
 
-class InvalidPatcherConfigurationException(message: String, e: Exception? = null) : Exception(message, e)
-
+class InvalidPatcherConfigurationException(message: String, e: Exception? = null) : DeploymentFailedException(message,e)

theodolite/src/main/kotlin/theodolite/util/TheodoliteException.kt

+package theodolite.util
+
+open class TheodoliteException (message: String, e: Exception? = null) : Exception(message,e)
\ No newline at end of file