Add basic support for loading resources via a so called resource set either...

Add basic support for loading resources via a so called resource set either from configmap or from file system

theodolite/crd/crd-benchmark.yaml

@@ -136,6 +136,40 @@ spec:
                           description: Determines if this topic should only be deleted after each experiement. For removeOnly topics the name can be a RegEx describing the topic.
                           type: boolean
                           default: false
+              appResourceSets:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    ConfigMapResourceSet:
+                      type: object
+                      properties:
+                        configmap:
+                          type: string
+                    FileSystemResourceSet:
+                      type: object
+                      properties:
+                        path:
+                          type: string
+              loadGenResourceSets:
+                type: array
+                items:
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    ConfigMapResourceSet:
+                      type: object
+                      properties:
+                        configmap:
+                          type: string
+                    FileSystemResourceSet:
+                      type: object
+                      properties:
+                        path:
+                          type: string
     additionalPrinterColumns:
     - name: Age
       type: date

theodolite/examples/operator/example-benchmark.yaml

@@ -35,4 +35,15 @@ spec:
         numPartitions: 40
         replicationFactor: 1
       - name: "theodolite-.*"
-        removeOnly: True
\ No newline at end of file
+        removeOnly: True
+        numPartitions: 0
+        replicationFactor: 0
+  appResourceSets:
+    - name: TestAppResources
+      FileSystemResourceSet:
+        path: ./config
+  loadGenResourceSets:
+    - name: RestGenResources
+      ConfigMapResourceSet:
+        configmap: "test-configmap"
+    
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt

+package theodolite.benchmark
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize
+import io.fabric8.kubernetes.api.model.KubernetesResource
+import io.fabric8.kubernetes.client.DefaultKubernetesClient
+import io.fabric8.kubernetes.client.NamespacedKubernetesClient
+import mu.KotlinLogging
+import theodolite.k8s.K8sResourceLoaderFromString
+import theodolite.util.YamlParserFromString
+import kotlin.math.log
+
+private val logger = KotlinLogging.logger {}
+
+
+
+@JsonDeserialize
+class ConfigMapResourceSet: ResourceSet {
+    lateinit var configmap: String
+    lateinit var files: List<String> // load all files, iff files is not set
+    private val namespace = "default"
+    private val client: NamespacedKubernetesClient = DefaultKubernetesClient().inNamespace(namespace) // TODO(load namespace from env var)
+    private val loader = K8sResourceLoaderFromString(client)
+
+
+    @OptIn(ExperimentalStdlibApi::class)
+    override fun getResourceSet(): List<Pair<String, KubernetesResource>> {
+        logger.info { "Load benchmark resources from configmap with name $configmap" }
+
+        var resources = client
+            .configMaps()
+            .withName(configmap)
+            .get()
+            .data
+
+
+        if (::files.isInitialized){
+            resources = resources
+                .filterKeys { files.contains(it) }
+        }
+
+        return resources
+            .map { Pair(
+                getKind(resource = it.value),
+                resources) }
+            .map { Pair(
+                it.first,
+                loader.loadK8sResource(it.first, it.second.values.first())) }
+    }
+
+    private fun getKind(resource: String): String {
+        logger.info { "1" }
+        val parser = YamlParserFromString()
+        val resoureceAsMap = parser.parse(resource, HashMap<String, String>()::class.java)
+        logger.info { "2" }
+
+        return try {
+            val kind = resoureceAsMap?.get("kind")!!
+            logger.info { "Kind is $kind" }
+            kind
+
+        } catch (e: Exception) {
+            logger.error { "Could not find field kind of Kubernetes resource: ${resoureceAsMap?.get("name")}" }
+            ""
+        }
+    }
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/benchmark/FileSystemResourceSet.kt

+package theodolite.benchmark
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize
+import io.fabric8.kubernetes.api.model.KubernetesResource
+import io.fabric8.kubernetes.client.DefaultKubernetesClient
+import mu.KotlinLogging
+import theodolite.k8s.K8sResourceLoader
+import theodolite.util.DeploymentFailedException
+import theodolite.util.YamlParserFromFile
+import java.io.File
+
+private val logger = KotlinLogging.logger {}
+
+
+@JsonDeserialize
+class FileSystemResourceSet: ResourceSet {
+    lateinit var path: String
+    lateinit var files: List<String>
+    private val parser = YamlParserFromFile()
+    private val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace("default")) // TODO(set namespace correctly)
+
+    override fun getResourceSet(): List<Pair<String, KubernetesResource>> {
+        logger.info { "Get fileSystem resource set $path" }
+
+
+        //if files is set ...
+        if(::files.isInitialized){
+            return files
+                .map { loadSingleResource(it)
+                }
+        }
+
+        return try {
+            File(path)
+                .list() !!
+                .map {
+                    loadSingleResource(it)
+                }
+        } catch (e: Exception) {
+            throw  DeploymentFailedException("Could not load files located in $path")
+        }
+    }
+
+    private fun loadSingleResource(resourceURL: String): Pair<String, KubernetesResource> {
+        val resourcePath = "$path/$resourceURL"
+        val kind = parser.parse(resourcePath, HashMap<String, String>()::class.java)?.get("kind")!!
+        val k8sResource = loader.loadK8sResource(kind, resourcePath)
+        return Pair(resourceURL, k8sResource)
+    }
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt

@@ -2,8 +2,6 @@ package theodolite.benchmark
 
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 import io.fabric8.kubernetes.api.model.KubernetesResource
-import io.fabric8.kubernetes.api.model.Namespaced
-import io.fabric8.kubernetes.client.CustomResource
 import io.fabric8.kubernetes.client.DefaultKubernetesClient
 import io.quarkus.runtime.annotations.RegisterForReflection
 import mu.KotlinLogging
@@ -11,6 +9,7 @@ import theodolite.k8s.K8sResourceLoader
 import theodolite.patcher.PatcherFactory
 import theodolite.util.*
 
+
 private val logger = KotlinLogging.logger {}
 
 private var DEFAULT_NAMESPACE = "default"
@@ -40,25 +39,19 @@ class KubernetesBenchmark: KubernetesResource, Benchmark{
     lateinit var resourceTypes: List<TypeName>
     lateinit var loadTypes: List<TypeName>
     lateinit var kafkaConfig: KafkaConfig
+    lateinit var appResourceSets: List<ResourceSets>
+    lateinit var loadGenResourceSets: List<ResourceSets>
     var namespace = System.getenv("NAMESPACE") ?: DEFAULT_NAMESPACE
     var path =  System.getenv("THEODOLITE_APP_RESOURCES") ?: "./config"
 
 
     /**
      * Loads [KubernetesResource]s.
-     * It first loads them via the [YamlParser] to check for their concrete type and afterwards initializes them using
+     * It first loads them via the [YamlParserFromFile] to check for their concrete type and afterwards initializes them using
      * the [K8sResourceLoader]
      */
-    private fun loadKubernetesResources(resources: List<String>): List<Pair<String, KubernetesResource>> {
-        val parser = YamlParser()
-        val loader = K8sResourceLoader(DefaultKubernetesClient().inNamespace(namespace))
-        return resources
-            .map { resource ->
-                val resourcePath = "$path/$resource"
-                val kind = parser.parse(resourcePath, HashMap<String, String>()::class.java)?.get("kind")!!
-                val k8sResource = loader.loadK8sResource(kind, resourcePath)
-                Pair(resource, k8sResource)
-            }
+    fun loadKubernetesResources(resourceSet: List<ResourceSets>): List<Pair<String, KubernetesResource>> {
+        return resourceSet.flatMap { it.loadResourceSet() }
     }
 
     /**
@@ -80,8 +73,8 @@ class KubernetesBenchmark: KubernetesResource, Benchmark{
         logger.info { "Using $namespace as namespace." }
         logger.info { "Using $path as resource path." }
 
-        val appResources = loadKubernetesResources(this.appResource)
-        val loadGenResources = loadKubernetesResources(this.loadGenResource)
+        val appResources = loadKubernetesResources(this.appResourceSets)
+        val loadGenResources = loadKubernetesResources(this.loadGenResourceSets)
 
         val patcherFactory = PatcherFactory()
 

theodolite/src/main/kotlin/theodolite/benchmark/ResourceSet.kt

+package theodolite.benchmark
+
+import io.fabric8.kubernetes.api.model.KubernetesResource
+
+interface ResourceSet {
+
+    fun getResourceSet(): List<Pair<String, KubernetesResource>>
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/benchmark/ResourceSets.kt

+package theodolite.benchmark
+
+import com.fasterxml.jackson.annotation.JsonInclude
+import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize
+import io.fabric8.kubernetes.api.model.KubernetesResource
+import io.fabric8.kubernetes.client.DefaultKubernetesClient
+import io.fabric8.kubernetes.client.NamespacedKubernetesClient
+import io.quarkus.runtime.annotations.RegisterForReflection
+import mu.KotlinLogging
+import theodolite.k8s.K8sResourceLoaderFromString
+import theodolite.util.DeploymentFailedException
+
+private val logger = KotlinLogging.logger {}
+
+@JsonDeserialize
+@RegisterForReflection
+@JsonInclude(JsonInclude.Include.NON_NULL)
+class ResourceSets: KubernetesResource {
+
+    @JsonProperty
+    lateinit var name: String
+
+    @JsonProperty("ConfigMapResourceSet")
+    val ConfigMapResourceSet: ConfigMapResourceSet? = null
+
+    @JsonProperty("FileSystemResourceSet")
+    val FileSystemResourceSet: FileSystemResourceSet? = null
+
+    fun loadResourceSet(): List<Pair<String, KubernetesResource>> {
+        logger.info { "LOAD" }
+        return try {
+            if (ConfigMapResourceSet != null) {
+                ConfigMapResourceSet.getResourceSet()
+            } else if (FileSystemResourceSet != null) {
+                FileSystemResourceSet.getResourceSet()
+            } else {
+                throw  DeploymentFailedException("could not load resourceSet.")
+            }
+        } catch (e: Exception) {
+            throw e
+        }
+
+    }
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/execution/TheodoliteYamlExecutor.kt

@@ -3,7 +3,7 @@ package theodolite.execution
 import mu.KotlinLogging
 import theodolite.benchmark.BenchmarkExecution
 import theodolite.benchmark.KubernetesBenchmark
-import theodolite.util.YamlParser
+import theodolite.util.YamlParserFromFile
 import kotlin.concurrent.thread
 import kotlin.system.exitProcess
 
@@ -26,7 +26,7 @@ private val logger = KotlinLogging.logger {}
  * @constructor Create empty Theodolite yaml executor
  */
 class TheodoliteYamlExecutor {
-    private val parser = YamlParser()
+    private val parser = YamlParserFromFile()
 
     fun start() {
         logger.info { "Theodolite started" }

theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt

@@ -124,7 +124,7 @@ class TheodoliteOperator {
         )
     }
 
-    private fun getBenchmarkClient(client: NamespacedKubernetesClient): MixedOperation<
+    fun getBenchmarkClient(client: NamespacedKubernetesClient): MixedOperation<
             BenchmarkCRD,
             KubernetesBenchmarkList,
             Resource<BenchmarkCRD>> {

theodolite/src/main/kotlin/theodolite/k8s/AbstractK8sLoader.kt

+package theodolite.k8s
+
+abstract class AbstractK8sLoader {
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoader.kt

@@ -7,7 +7,7 @@ import io.fabric8.kubernetes.api.model.apps.Deployment
 import io.fabric8.kubernetes.client.NamespacedKubernetesClient
 import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext
 import mu.KotlinLogging
-import theodolite.util.YamlParser
+import theodolite.util.YamlParserFromFile
 
 private val logger = KotlinLogging.logger {}
 
@@ -37,7 +37,7 @@ class K8sResourceLoader(private val client: NamespacedKubernetesClient) {
    private fun loadCustomResourceWrapper(path: String, context: CustomResourceDefinitionContext): CustomResourceWrapper {
        return loadGenericResource(path) {
            CustomResourceWrapper(
-               YamlParser().parse(
+               YamlParserFromFile().parse(
                    path,
                    HashMap<String, String>()::class.java
                )!!,

theodolite/src/main/kotlin/theodolite/k8s/K8sResourceLoaderFromString.kt

+package theodolite.k8s
+
+import io.fabric8.kubernetes.api.model.ConfigMap
+import io.fabric8.kubernetes.api.model.KubernetesResource
+import io.fabric8.kubernetes.api.model.Service
+import io.fabric8.kubernetes.api.model.apps.Deployment
+import io.fabric8.kubernetes.client.NamespacedKubernetesClient
+import io.fabric8.kubernetes.client.utils.Serialization
+import mu.KotlinLogging
+import theodolite.util.YamlParserFromString
+import java.io.ByteArrayInputStream
+private val logger = KotlinLogging.logger {}
+
+
+
+class K8sResourceLoaderFromString(private val client: NamespacedKubernetesClient) {
+
+    @OptIn(ExperimentalStdlibApi::class)
+    fun loadK8sResource(kind: String, resourceString: String): KubernetesResource {
+
+        return when (kind) {
+            "Deployment" -> loadDeployment(resourceString)
+            "Service" -> loadService(resourceString)
+            //"ServiceMonitor" -> loadServiceMonitor(resourceString) // TODO(Add support for custom resources)
+            "ConfigMap" -> loadConfigmap(resourceString)
+            "StatefulSet" -> loadStatefulSet(resourceString)
+            //"Execution" -> loadExecution(resourceString)
+            //"Benchmark" -> loadBenchmark(resourceString)
+            else -> {
+                logger.error { "Error during loading of unspecified resource Kind" }
+                throw java.lang.IllegalArgumentException("error while loading resource with kind: $kind")
+            }
+        }
+    }
+
+    /**
+     * Generic helper function to load a resource.
+     * @param path of the resource
+     * @param f function that is applied to the resource.
+     * @throws IllegalArgumentException If the resource could not be loaded.
+     */
+    @OptIn(ExperimentalStdlibApi::class)
+    private fun <T> loadGenericResource(resourceString: String, f: (ByteArrayInputStream) -> T): T {
+        val stream = ByteArrayInputStream(resourceString.encodeToByteArray())
+        var resource: T? = null
+
+        try {
+            resource = f(stream)
+        } catch (e: Exception) {
+            logger.warn { "You potentially  misspelled the path: ....1" }
+            logger.warn { e }
+        }
+
+        if (resource == null) {
+            throw IllegalArgumentException("The Resource: ....1 could not be loaded")
+        }
+        return resource
+    }
+
+
+    @OptIn(ExperimentalStdlibApi::class)
+    private fun loadService(resourceStream: String): KubernetesResource {
+
+        //logger.info { resourceStream }
+
+        val stream = ByteArrayInputStream(resourceStream.encodeToByteArray())
+        //val test = Serialization.unmarshal<Service>(stream, Service::class.java)
+        //logger.info { test }
+        // return test
+        logger.info { "Test" }
+        //val parser = YamlParserFromString()
+        //val resoureceAsMap = parser.parse(resourceStream, HashMap<String, String>()::class.java)
+        //val loadedSvc: Service = client.services().load(stream).get()
+        //logger.info { "loadedSvc" }
+        //return loadedSvc
+        //logger.info { "try to load service" }
+        return loadGenericResource(resourceStream) { x: ByteArrayInputStream -> client.services().load(x).get() }
+    }
+
+    private fun loadDeployment(path: String): Deployment {
+        return loadGenericResource(path) { x: ByteArrayInputStream -> client.apps().deployments().load(x).get() }
+    }
+
+    private fun loadConfigmap(path: String): ConfigMap {
+        return loadGenericResource(path) { x: ByteArrayInputStream -> client.configMaps().load(x).get() }
+    }
+
+    private fun loadStatefulSet(path: String): KubernetesResource {
+        return loadGenericResource(path) { x: ByteArrayInputStream -> client.apps().statefulSets().load(x).get() }
+    }
+}
\ No newline at end of file

theodolite/src/main/kotlin/theodolite/util/YamlParser.kt → theodolite/src/main/kotlin/theodolite/util/YamlParserFromFile.kt

@@ -9,7 +9,7 @@ import java.io.InputStream
 /**
  * The YamlParser parses a YAML file
  */
-class YamlParser : Parser {
+class YamlParserFromFile : Parser {
     override fun <T> parse(path: String, E: Class<T>): T? {
         val input: InputStream = FileInputStream(File(path))
         val parser = Yaml(Constructor(E))

theodolite/src/main/kotlin/theodolite/util/YamlParserFromString.kt

+package theodolite.util
+
+import org.yaml.snakeyaml.Yaml
+import org.yaml.snakeyaml.constructor.Constructor
+import java.io.File
+import java.io.FileInputStream
+import java.io.InputStream
+
+/**
+ * The YamlParser parses a YAML string
+ */
+class YamlParserFromString : Parser {
+    override fun <T> parse(fileString: String, E: Class<T>): T? {
+        val parser = Yaml(Constructor(E))
+        return parser.loadAs(fileString, E)
+    }
+}

theodolite/src/test/kotlin/theodolite/execution/operator/testTest.kt

+package theodolite.execution.operator
+
+class testTest {
+}
\ No newline at end of file