Theodolite error message with unknown origin
Theodolite threw the following exception after running a benchmark after ~ 90min. Error is not reproducible. It seems that the operator runs in any timeout problem. I am not sure what is causing this problem.
Thread-53128] WARN theodolite.execution.operator.AbstractStateHandler - Status cannot be set for resource exec-flsc-attachments-file-size-lin-run1.
io.fabric8.kubernetes.client.KubernetesClientException: Operation: [get] for kind: [execution] with name: [exec-flsc-attachments-file-size-lin-run1] in namespace: [theodolite-stu126940] failed.
at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:130)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:189)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:151)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:83)
at theodolite.execution.operator.AbstractStateHandler.setState(AbstractStateHandler.kt:25)
at theodolite.execution.operator.ExecutionStateHandler.updateDurationState(ExecutionStateHandler.kt:32)
at theodolite.execution.operator.ExecutionStateHandler.startDurationStateTimer$lambda-1(ExecutionStateHandler.kt:43)
at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.net.SocketTimeoutException: timeout
at okhttp3.internal.http2.Http2Stream$StreamTimeout.newTimeoutException(Http2Stream.java:678)
at okhttp3.internal.http2.Http2Stream$StreamTimeout.exitAndThrowIfTimedOut(Http2Stream.java:686)
at okhttp3.internal.http2.Http2Stream.takeHeaders(Http2Stream.java:154)
at okhttp3.internal.http2.Http2ExchangeCodec.readResponseHeaders(Http2ExchangeCodec.java:136)
at okhttp3.internal.connection.Exchange.readResponseHeaders(Exchange.java:115)
at okhttp3.internal.http.CallServerInterceptor.intercept(CallServerInterceptor.java:94)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142)
at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:43)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117)
at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:94)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117)
at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142)
at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:88)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117)
at io.fabric8.kubernetes.client.okhttp.OkHttpClientBuilderImpl$InteceptorAdapter.intercept(OkHttpClientBuilderImpl.java:62)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117)
at io.fabric8.kubernetes.client.okhttp.OkHttpClientBuilderImpl$InteceptorAdapter.intercept(OkHttpClientBuilderImpl.java:62)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117)
at io.fabric8.kubernetes.client.okhttp.OkHttpClientBuilderImpl$InteceptorAdapter.intercept(OkHttpClientBuilderImpl.java:62)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117)
at io.fabric8.kubernetes.client.okhttp.OkHttpClientBuilderImpl$InteceptorAdapter.intercept(OkHttpClientBuilderImpl.java:62)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142)
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117)
at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:229)
at okhttp3.RealCall.execute(RealCall.java:81)
at io.fabric8.kubernetes.client.okhttp.OkHttpClientImpl.send(OkHttpClientImpl.java:138)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.retryWithExponentialBackoff(OperationSupport.java:574)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:553)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:518)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:487)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:457)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleGet(BaseOperation.java:698)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:184)
... 6 more
[Thread-1] INFO theodolite.util.IOHandler - Wrote txt file: /deployments/results/exp2-result to /deployments/results/exp2-result.
[Thread-1] INFO theodolite.util.IOHandler - Wrote CSV file: /deployments/results/exp2_demand to /deployments/results/exp2_demand.csv.
[Thread-1] ERROR theodolite.execution.operator.TheodoliteController - Failure while executing execution exec-flsc-attachments-file-size-lin-run1 with benchmark benchmark-attachments-inf.
io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: GET at: https://10.233.0.1/apis/theodolite.com/v1/namespaces/theodolite-stu126940/executions. Message: etcdserver: leader changed. Received status: Status(apiVersion=v1, code=500, details=null, kind=Status, message=etcdserver: leader changed, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=null, status=Failure, additionalProperties={}).
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:682)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:661)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:612)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:555)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:518)
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:502)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.listRequestHelper(BaseOperation.java:133)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:415)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:404)
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:83)
at theodolite.execution.operator.EventCreator.buildObjectReference(EventCreator.kt:46)
at theodolite.execution.operator.EventCreator.createEvent(EventCreator.kt:21)
at theodolite.execution.BenchmarkExecutorImpl.runSingleExperiment(BenchmarkExecutorImpl.kt:105)
at theodolite.execution.BenchmarkExecutorImpl.runExperiment(BenchmarkExecutorImpl.kt:49)
at theodolite.strategies.searchstrategy.LinearSearch.findSuitableResource(LinearSearch.kt:21)
at theodolite.strategies.searchstrategy.CompositeStrategy.findSuitableResource(CompositeStrategy.kt:28)
at theodolite.execution.TheodoliteExecutor.run(TheodoliteExecutor.kt:132)
at theodolite.execution.operator.TheodoliteController.runExecution(TheodoliteController.kt:92)
at theodolite.execution.operator.TheodoliteController.reconcile(TheodoliteController.kt:55)
at theodolite.execution.operator.TheodoliteController.run(TheodoliteController.kt:42)
at theodolite.execution.operator.TheodoliteOperator.startOperator(TheodoliteOperator.kt:78)
at theodolite.execution.operator.TheodoliteOperator.access$startOperator(TheodoliteOperator.kt:30)
at theodolite.execution.operator.TheodoliteOperator$start$1.invoke(TheodoliteOperator.kt:44)
at theodolite.execution.operator.TheodoliteOperator$start$1.invoke(TheodoliteOperator.kt:44)
at theodolite.execution.operator.LeaderElector.getLeadership$lambda-4$lambda-1$lambda-0(LeaderElector.kt:33)
at java.base/java.lang.Thread.run(Thread.java:829)