diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt index cbe59dd2286a599b96b0a38855ed08879f122c71..86b6e9416ed34c93218cc40a7d8fdc0867c871d1 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/BenchmarkExecutor.kt @@ -61,7 +61,7 @@ abstract class BenchmarkExecutor( } } - logger.info { "Exucutor shutdown gracefully" } + logger.debug { "Executor shutdown gracefully" } } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt index 9ca7e70f6ecd5ed30120940be8de965c73f79a6a..fe4b983bca71dd5801f5a421cf44583f085cec90 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/Shutdown.kt @@ -20,7 +20,7 @@ class Shutdown(private val benchmarkExecution: BenchmarkExecution, private val b res = Resource(0, emptyList()), configurationOverrides = benchmarkExecution.configOverrides ) - logger.info { "Teardown the everything deployed" } + logger.info { "Teardown everything deployed" } deployment.teardown() logger.info { "Teardown completed" } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteController.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteController.kt index 77f667188d0b1ef46d91ac493cd85a388e0b71fc..ec5f7763788f2fd37f2430be6f526f0cc2f3fa5e 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteController.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteController.kt @@ -42,8 +42,8 @@ class TheodoliteController( if (::executor.isInitialized && executor.getExecution().name == newExecution.metadata.name) { logger.info { "restart current benchmark with new version" } executor.stop() - executor = TheodoliteExecutor(config = newExecution, kubernetesBenchmark = executor.getBenchmark()) - executor.run() + sleep(2000) + startBenchmark(execution = newExecution, benchmark = executor.getBenchmark()) } else { onDelete(oldExecution, false) onAdd(newExecution) @@ -70,10 +70,11 @@ class TheodoliteController( override fun onUpdate(oldBenchmark: KubernetesBenchmark, newBenchmark: KubernetesBenchmark) { logger.info { "Update benchmark ${newBenchmark.metadata.name}" } if (::executor.isInitialized && executor.getBenchmark().name == oldBenchmark.metadata.name) { + logger.info { "restart current benchmark with new version" } executor.stop() - executor = TheodoliteExecutor(config = executor.getExecution(), kubernetesBenchmark = newBenchmark) - executor.run() + sleep(2000) + startBenchmark(execution = executor.getExecution(), benchmark = newBenchmark) } else { onAdd(newBenchmark) } @@ -94,8 +95,10 @@ class TheodoliteController( while (true) { try { reconcile() - logger.info { "Theodolite is waiting for new jobs" } - sleep(1000) + if (this::executor.isInitialized && !executor.isRunning) { + logger.info { "Theodolite is waiting for new jobs" } + sleep(1000) + } } catch (e: InterruptedException) { logger.error { "Execution interrupted with error: $e" } } @@ -114,21 +117,29 @@ class TheodoliteController( logger.debug { "No benchmark found for execution ${execution.benchmark}" } sleep(1000) } else { - logger.info { "Start execution ${execution.name} with benchmark ${benchmark.name}" } - executor = TheodoliteExecutor(config = execution, kubernetesBenchmark = benchmark) - executor.run() - // wait until executions is deleted - try { - client.customResource(executionContext).delete(client.namespace, execution.metadata.name) - while (executionsQueue.contains(execution)) { - logger.info { "sleep" } - sleep(1000) - } - } catch (e: Exception) { - logger.error { "Error while delete current execution: $e" } - } - logger.info { "Execution is finally stopped for execution ${execution.name}" } + startBenchmark(execution, benchmark) + } + } + } + + + fun startBenchmark(execution: BenchmarkExecution, benchmark: KubernetesBenchmark) { + + logger.info { "Start execution ${execution.name} with benchmark ${benchmark.name}" } + executor = TheodoliteExecutor(config = execution, kubernetesBenchmark = benchmark) + executor.run() + // wait until execution is deleted + try { + client.customResource(executionContext).delete(client.namespace, execution.metadata.name) + sleep(1000) + while (executionsQueue.contains(execution)) { + sleep(2000) + logger.info { "Delete of execution: ${execution.name} failed. Retrying in 2 second." } + client.customResource(executionContext).delete(client.namespace, execution.metadata.name) } + } catch (e: Exception) { + logger.error { "Error while delete current execution: $e" } } + logger.info { "Execution of ${execution.name} is finally stopped" } } } diff --git a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt index e831bcc1defd7aca6207a297dc0a25429ba65044..2ed356a797d4f90d09168bbc5cee4ad1b928464a 100644 --- a/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt +++ b/theodolite-quarkus/src/main/kotlin/theodolite/execution/TheodoliteExecutor.kt @@ -103,11 +103,11 @@ class TheodoliteExecutor( } } logger.info { "Stop Thread" } + isRunning = false } } fun stop() { - // TODO call shutdown hook isRunning = false try { executor.stop()