diff --git a/docs/_config.yml b/docs/_config.yml index 0d2a1aa774a83347c80b538a97d5dbfa1b7639b3..95cc5a635494842f1894c572cb117aafb3bb6810 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -3,7 +3,7 @@ description: >- Theodolite is a framework for benchmarking the horizontal and vertical scalability of cloud-native applications. -remote_theme: pmarsceill/just-the-docs +remote_theme: just-the-docs/just-the-docs aux_links: "Theodolite on GitHub": - "//github.com/cau-se/theodolite" diff --git a/docs/theodolite-benchmarks/load-generator.md b/docs/theodolite-benchmarks/load-generator.md index 5ae10d16a50aaa16a76975d8127ef379508b1a37..a41c97d52f62f399c9289a15a64991d0fed228ce 100644 --- a/docs/theodolite-benchmarks/load-generator.md +++ b/docs/theodolite-benchmarks/load-generator.md @@ -56,6 +56,7 @@ The prebuilt container images can be configured with the following environment v | `KAFKA_BUFFER_MEMORY` | Value for the Kafka producer configuration: [`buffer.memory`](https://kafka.apache.org/documentation/#producerconfigs_buffer.memory) Only used if Kafka is set as `TARGET`. | see Kafka producer config: [`buffer.memory`](https://kafka.apache.org/documentation/#producerconfigs_buffer.memory) | | `HTTP_URL` | The URL the load generator should post messages to. Only used if HTTP is set as `TARGET`. | | | `HTTP_ASYNC` | Whether the load generator should send HTTP messages asynchronously. Only used if HTTP is set as `TARGET`. | `false` | +| `HTTP_TIMEOUT_MS` | Timeout in milliseconds for sending HTTP messages. Only used if HTTP is set as `TARGET`. | 1000 | | `PUBSUB_INPUT_TOPIC` | The Google Cloud Pub/Sub topic to write messages to. Only used if Pub/Sub is set as `TARGET`. | input | | `PUBSUB_PROJECT` | The Google Cloud this Pub/Sub topic is associated with. Only used if Pub/Sub is set as `TARGET`. | | | `PUBSUB_EMULATOR_HOST` | A Pub/Sub emulator host. Only used if Pub/Sub is set as `TARGET`. | | diff --git a/helm/preconfigs/minimal.yaml b/helm/preconfigs/minimal.yaml index 80a83f06cc9838e01f812e730932b9b79d947161..3f2dba0b55b729654125377996381718d7c15eb4 100644 --- a/helm/preconfigs/minimal.yaml +++ b/helm/preconfigs/minimal.yaml @@ -1,15 +1,15 @@ -cp-helm-charts: - cp-zookeeper: - servers: 1 - - cp-kafka: - brokers: 1 - configurationOverrides: - offsets.topic.replication.factor: "1" - operator: sloChecker: droppedRecordsKStreams: enabled: false resultsVolume: enabled: false + +strimzi: + kafka: + replicas: 1 + config: + "offsets.topic.replication.factor": "1" + zookeeper: + replicas: 1 + \ No newline at end of file diff --git a/helm/templates/theodolite/role.yaml b/helm/templates/theodolite/role.yaml index ba5a223b6527df94b64fac3574ee5f90fdb3903b..8b3961a33bd90f81af29b5adde9da449c6a462d8 100644 --- a/helm/templates/theodolite/role.yaml +++ b/helm/templates/theodolite/role.yaml @@ -45,6 +45,16 @@ rules: - list - create - get + - apiGroups: + - kafka.strimzi.io + resources: + - kafkatopics + verbs: + - delete + - list + - get + - create + - update {{- if .Values.operator.enabled }} - apiGroups: - theodolite.com diff --git a/helm/values.yaml b/helm/values.yaml index 0f1d4790b6d03d42f6ccc65a5156eb70999867e6..188332ef148e3e0e5a8b995fde3c8921581f718b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -181,7 +181,7 @@ strimzi: enabled: true nodeSelector: {} topicOperator: - enabled: false + enabled: true ### diff --git a/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/AbstractPipelineFactory.java b/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/AbstractPipelineFactory.java index 4976f46e231c472599f85a72f698e26a09cbc860..f6dc64bb2c8f4deb0df6e48db23b0d62c6d86279 100644 --- a/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/AbstractPipelineFactory.java +++ b/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/AbstractPipelineFactory.java @@ -48,10 +48,13 @@ public abstract class AbstractPipelineFactory { final Map<String, Object> consumerConfig = new HashMap<>(); consumerConfig.put( ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, - this.config.getString(ConfigurationKeys.ENABLE_AUTO_COMMIT_CONFIG)); + this.config.getString(ConfigurationKeys.ENABLE_AUTO_COMMIT)); + consumerConfig.put( + ConsumerConfig.MAX_POLL_RECORDS_CONFIG, + this.config.getString(ConfigurationKeys.MAX_POLL_RECORDS)); consumerConfig.put( ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, - this.config.getString(ConfigurationKeys.AUTO_OFFSET_RESET_CONFIG)); + this.config.getString(ConfigurationKeys.AUTO_OFFSET_RESET)); consumerConfig.put( AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, this.config.getString(ConfigurationKeys.SCHEMA_REGISTRY_URL)); diff --git a/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/BeamService.java b/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/BeamService.java index dd410f8d52e269a863b5d6dab62196c0d9690c98..a4a8f69d74f32697d8e43d58bc5765631fea63de 100644 --- a/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/BeamService.java +++ b/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/BeamService.java @@ -46,7 +46,7 @@ public class BeamService { * Start this microservice, by running the underlying Beam pipeline. */ public void run() { - LOGGER.info("Construct Beam pipeline with pipeline options: {}", + LOGGER.info("Constructing Beam pipeline with pipeline options: {}", this.pipelineOptions.toString()); final Pipeline pipeline = this.pipelineFactory.create(this.pipelineOptions); LOGGER.info("Starting BeamService {}.", this.applicationName); diff --git a/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/ConfigurationKeys.java b/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/ConfigurationKeys.java index 487b8de00c35bbe28961f29de7ba0bb9d57e98ec..c22c164f62ad22d3c18add75ad5115fd15fb8f14 100644 --- a/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/ConfigurationKeys.java +++ b/theodolite-benchmarks/beam-commons/src/main/java/rocks/theodolite/benchmarks/commons/beam/ConfigurationKeys.java @@ -33,16 +33,17 @@ public final class ConfigurationKeys { // BEAM - public static final String ENABLE_AUTO_COMMIT_CONFIG = "enable.auto.commit.config"; + public static final String ENABLE_AUTO_COMMIT = "enable.auto.commit"; - public static final String AUTO_OFFSET_RESET_CONFIG = "auto.offset.reset.config"; + public static final String MAX_POLL_RECORDS = "max.poll.records"; + + public static final String AUTO_OFFSET_RESET = "auto.offset.reset"; public static final String SPECIFIC_AVRO_READER = "specific.avro.reader"; - public static final String TRIGGER_INTERVAL = "trigger.interval"; + public static final String TRIGGER_INTERVAL = "trigger.interval"; - private ConfigurationKeys() { - } + private ConfigurationKeys() {} } diff --git a/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/ConfigurationKeys.java b/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/ConfigurationKeys.java index efb7db61cc4c81ec2d1ffd49141d6d70a23dacaa..eb80d25eb327f2e3dc10dc2977131ac7edfef69d 100644 --- a/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/ConfigurationKeys.java +++ b/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/ConfigurationKeys.java @@ -43,6 +43,8 @@ public final class ConfigurationKeys { public static final String HTTP_ASYNC = "HTTP_ASYNC"; + public static final String HTTP_TIMEOUT_MS = "HTTP_TIMEOUT_MS"; + public static final String PUBSUB_INPUT_TOPIC = "PUBSUB_INPUT_TOPIC"; public static final String PUBSUB_PROJECT = "PUBSUB_PROJECT"; diff --git a/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/EnvVarLoadGeneratorFactory.java b/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/EnvVarLoadGeneratorFactory.java index 29ede821eefe171f377d58fce8d98eee28e8a277..ae9a6d4220ceaec091a0a2fb49fb82f16fdbb42e 100644 --- a/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/EnvVarLoadGeneratorFactory.java +++ b/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/EnvVarLoadGeneratorFactory.java @@ -122,7 +122,10 @@ class EnvVarLoadGeneratorFactory { final boolean async = Boolean.parseBoolean(Objects.requireNonNullElse( System.getenv(ConfigurationKeys.HTTP_ASYNC), Boolean.toString(LoadGenerator.HTTP_ASYNC_DEFAULT))); - recordSender = new HttpRecordSender<>(url, async); + final long timeoutMs = Integer.parseInt(Objects.requireNonNullElse( + System.getenv(ConfigurationKeys.HTTP_TIMEOUT_MS), + Long.toString(LoadGenerator.HTTP_TIMEOUT_MS_DEFAULT))); + recordSender = new HttpRecordSender<>(url, async, Duration.ofMillis(timeoutMs)); LOGGER.info("Use HTTP server as target with URL '{}' and asynchronously: '{}'.", url, async); } else if (target == LoadGeneratorTarget.PUBSUB) { final String project = System.getenv(ConfigurationKeys.PUBSUB_PROJECT); @@ -139,7 +142,7 @@ class EnvVarLoadGeneratorFactory { LOGGER.info("Use Pub/Sub as target with project {} and topic '{}'.", project, inputTopic); recordSender = TitanPubSubSenderFactory.forPubSubConfig(project, inputTopic); } else { - throw new IllegalStateException("Neither an emulator host nor a project was provided."); + throw new IllegalStateException("Neither an emulator host nor a project was provided."); } } else { // Should never happen diff --git a/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/HttpRecordSender.java b/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/HttpRecordSender.java index 124f11a979f3afad5507db86c68e9eeb42c64eb6..77706d824808132eaa7212194de0d69c346e4eba 100644 --- a/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/HttpRecordSender.java +++ b/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/HttpRecordSender.java @@ -24,7 +24,7 @@ public class HttpRecordSender<T extends SpecificRecord> implements RecordSender< private static final int HTTP_OK = 200; - private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(1); + private static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofSeconds(1); private static final Logger LOGGER = LoggerFactory.getLogger(HttpRecordSender.class); @@ -36,6 +36,8 @@ public class HttpRecordSender<T extends SpecificRecord> implements RecordSender< private final boolean async; + private final Duration connectionTimeout; + private final List<Integer> validStatusCodes; /** @@ -44,7 +46,7 @@ public class HttpRecordSender<T extends SpecificRecord> implements RecordSender< * @param uri the {@link URI} records should be sent to */ public HttpRecordSender(final URI uri) { - this(uri, false, List.of(HTTP_OK)); + this(uri, false, DEFAULT_CONNECTION_TIMEOUT); } /** @@ -52,9 +54,10 @@ public class HttpRecordSender<T extends SpecificRecord> implements RecordSender< * * @param uri the {@link URI} records should be sent to * @param async whether HTTP requests should be sent asynchronous + * @param connectionTimeout timeout for the HTTP connection */ - public HttpRecordSender(final URI uri, final boolean async) { - this(uri, async, List.of(HTTP_OK)); + public HttpRecordSender(final URI uri, final boolean async, final Duration connectionTimeout) { + this(uri, async, connectionTimeout, List.of(HTTP_OK)); } /** @@ -62,12 +65,17 @@ public class HttpRecordSender<T extends SpecificRecord> implements RecordSender< * * @param uri the {@link URI} records should be sent to * @param async whether HTTP requests should be sent asynchronous + * @param connectionTimeout timeout for the HTTP connection * @param validStatusCodes a list of HTTP status codes which are considered as successful */ - public HttpRecordSender(final URI uri, final boolean async, + public HttpRecordSender( + final URI uri, + final boolean async, + final Duration connectionTimeout, final List<Integer> validStatusCodes) { this.uri = uri; this.async = async; + this.connectionTimeout = connectionTimeout; this.validStatusCodes = validStatusCodes; } @@ -76,7 +84,7 @@ public class HttpRecordSender<T extends SpecificRecord> implements RecordSender< final String json = this.gson.toJson(message); final HttpRequest request = HttpRequest.newBuilder() .uri(this.uri) - .timeout(CONNECTION_TIMEOUT) + .timeout(this.connectionTimeout) .POST(HttpRequest.BodyPublishers.ofString(json)) .build(); final BodyHandler<Void> bodyHandler = BodyHandlers.discarding(); @@ -98,8 +106,10 @@ public class HttpRecordSender<T extends SpecificRecord> implements RecordSender< if (this.isSync()) { try { result.get(); - } catch (InterruptedException | ExecutionException e) { + } catch (final InterruptedException e) { LOGGER.error("Couldn't get result for request to {}.", this.uri, e); + } catch (final ExecutionException e) { // NOPMD + // Do nothing, Exception is already handled } } } diff --git a/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/LoadGenerator.java b/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/LoadGenerator.java index 27edb97efc335400acf1d6244db0ce384ee20f59..4be9b5695a54dedac6df78e6ceb8230752301e22 100644 --- a/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/LoadGenerator.java +++ b/theodolite-benchmarks/load-generator-commons/src/main/java/rocks/theodolite/benchmarks/loadgenerator/LoadGenerator.java @@ -18,6 +18,7 @@ public final class LoadGenerator { // Target: HTTP public static final String HTTP_URI_DEFAULT = "http://localhost:8080"; public static final boolean HTTP_ASYNC_DEFAULT = false; + public static final long HTTP_TIMEOUT_MS_DEFAULT = 1_000; // Target: Kafka public static final String SCHEMA_REGISTRY_URL_DEFAULT = "http://localhost:8081"; public static final String KAFKA_TOPIC_DEFAULT = "input"; // NOCS diff --git a/theodolite-benchmarks/load-generator-commons/src/test/java/rocks/theodolite/benchmarks/loadgenerator/HttpRecordSenderTest.java b/theodolite-benchmarks/load-generator-commons/src/test/java/rocks/theodolite/benchmarks/loadgenerator/HttpRecordSenderTest.java index 7c565ace82698bf47f6b3711a28e08f87e8e412b..731dda6c74fd3cd6d74771f95896c2260ce6df29 100644 --- a/theodolite-benchmarks/load-generator-commons/src/test/java/rocks/theodolite/benchmarks/loadgenerator/HttpRecordSenderTest.java +++ b/theodolite-benchmarks/load-generator-commons/src/test/java/rocks/theodolite/benchmarks/loadgenerator/HttpRecordSenderTest.java @@ -46,4 +46,22 @@ public class HttpRecordSenderTest { .withRequestBody(equalTo(expectedJson))); // toJson } + @Test + public void testTimeout() { + this.wireMockRule.stubFor( + post(urlPathEqualTo("/")) + .willReturn( + aResponse() + .withFixedDelay(2_000) + .withStatus(200) + .withBody("received"))); + + final ActivePowerRecord record = new ActivePowerRecord("my-id", 12345L, 12.34); + this.httpRecordSender.send(record); + + final String expectedJson = "{\"identifier\":\"my-id\",\"timestamp\":12345,\"valueInW\":12.34}"; + verify(exactly(1), postRequestedFor(urlEqualTo("/")) + .withRequestBody(equalTo(expectedJson))); // toJson + } + } diff --git a/theodolite-benchmarks/uc1-beam/src/main/java/rocks/theodolite/benchmarks/uc1/beam/PipelineFactory.java b/theodolite-benchmarks/uc1-beam/src/main/java/rocks/theodolite/benchmarks/uc1/beam/PipelineFactory.java index 32658a21b8b80fddb5baf58002a701e8e35b542e..1f35d592ed9b2b1507eb5c30090d392d37ed7c1e 100644 --- a/theodolite-benchmarks/uc1-beam/src/main/java/rocks/theodolite/benchmarks/uc1/beam/PipelineFactory.java +++ b/theodolite-benchmarks/uc1-beam/src/main/java/rocks/theodolite/benchmarks/uc1/beam/PipelineFactory.java @@ -9,6 +9,7 @@ import org.apache.beam.sdk.transforms.Values; import org.apache.commons.configuration2.Configuration; import rocks.theodolite.benchmarks.commons.beam.AbstractPipelineFactory; import rocks.theodolite.benchmarks.commons.beam.kafka.KafkaActivePowerTimestampReader; +import rocks.theodolite.benchmarks.uc1.beam.firestore.FirestoreOptionsExpander; import titan.ccp.model.records.ActivePowerRecord; /** @@ -17,6 +18,8 @@ import titan.ccp.model.records.ActivePowerRecord; public class PipelineFactory extends AbstractPipelineFactory { public static final String SINK_TYPE_KEY = "sink.type"; + + private final SinkType sinkType = SinkType.from(this.config.getString(SINK_TYPE_KEY)); public PipelineFactory(final Configuration configuration) { super(configuration); @@ -31,17 +34,18 @@ public class PipelineFactory extends AbstractPipelineFactory { // final PubsubOptions pubSubOptions = options.as(PubsubOptions.class); // pubSubOptions.setPubsubRootUrl("http://" + pubSubEmulatorHost); // } + if (this.sinkType == SinkType.FIRESTORE) { + FirestoreOptionsExpander.expandOptions(options); + } } @Override protected void constructPipeline(final Pipeline pipeline) { - final SinkType sinkType = SinkType.from(this.config.getString(SINK_TYPE_KEY)); - final KafkaActivePowerTimestampReader kafkaReader = super.buildKafkaReader(); pipeline.apply(kafkaReader) .apply(Values.create()) - .apply(sinkType.create(this.config)); + .apply(this.sinkType.create(this.config)); } @Override diff --git a/theodolite-benchmarks/uc1-beam/src/main/java/rocks/theodolite/benchmarks/uc1/beam/firestore/FirestoreOptionsExpander.java b/theodolite-benchmarks/uc1-beam/src/main/java/rocks/theodolite/benchmarks/uc1/beam/firestore/FirestoreOptionsExpander.java new file mode 100644 index 0000000000000000000000000000000000000000..0447450b45b971f96e2f2cbb7ce91f78604d5a23 --- /dev/null +++ b/theodolite-benchmarks/uc1-beam/src/main/java/rocks/theodolite/benchmarks/uc1/beam/firestore/FirestoreOptionsExpander.java @@ -0,0 +1,34 @@ +package rocks.theodolite.benchmarks.uc1.beam.firestore; + +import java.io.IOException; +import org.apache.beam.sdk.extensions.gcp.options.GcpOptions; +import org.apache.beam.sdk.options.PipelineOptions; + +/** + * Provides a method to expand {@link PipelineOptions} for Firestore. + */ +public final class FirestoreOptionsExpander { + + private FirestoreOptionsExpander() {} + + /** + * Expand {@link PipelineOptions} by special options required for Firestore derived from a default + * configuration. + * + * @param options {@link PipelineOptions} to be expanded. + */ + public static void expandOptions(final PipelineOptions options) { + final GcpOptions firestoreOptions = options.as(GcpOptions.class); + final FirestoreConfig firestoreConfig = getFirestoreConfig(); + firestoreOptions.setProject(firestoreConfig.getProjectId()); + } + + private static FirestoreConfig getFirestoreConfig() { + try { + return FirestoreConfig.createFromDefaults(); + } catch (final IOException e) { + throw new IllegalStateException("Cannot create Firestore configuration.", e); + } + } + +} diff --git a/theodolite-benchmarks/uc1-beam/src/main/resources/META-INF/application.properties b/theodolite-benchmarks/uc1-beam/src/main/resources/META-INF/application.properties index b785d698cd59a31bff7e9cffc21ca1d877f037fe..e9de96c0df34b1254a8ec9886586e163999c7c6e 100644 --- a/theodolite-benchmarks/uc1-beam/src/main/resources/META-INF/application.properties +++ b/theodolite-benchmarks/uc1-beam/src/main/resources/META-INF/application.properties @@ -14,6 +14,7 @@ num.threads=1 commit.interval.ms=1000 cache.max.bytes.buffering=-1 -specific.avro.reader=True -enable.auto.commit.config=True -auto.offset.reset.config=earliest +specific.avro.reader=true +enable.auto.commit=true +max.poll.records=500 +auto.offset.reset=earliest diff --git a/theodolite-benchmarks/uc2-beam/src/main/resources/META-INF/application.properties b/theodolite-benchmarks/uc2-beam/src/main/resources/META-INF/application.properties index 1545a0f6630c8ea51d694f4056ca3aa750463f5b..c6672125a8b6a074cb7eca31bd90700cd4da736a 100644 --- a/theodolite-benchmarks/uc2-beam/src/main/resources/META-INF/application.properties +++ b/theodolite-benchmarks/uc2-beam/src/main/resources/META-INF/application.properties @@ -12,6 +12,7 @@ num.threads=1 commit.interval.ms=1000 cache.max.bytes.buffering=-1 -specific.avro.reader=True -enable.auto.commit.config=True -auto.offset.reset.config=earliest \ No newline at end of file +specific.avro.reader=true +enable.auto.commit=true +max.poll.records=500 +auto.offset.reset=earliest \ No newline at end of file diff --git a/theodolite-benchmarks/uc3-beam/src/main/resources/META-INF/application.properties b/theodolite-benchmarks/uc3-beam/src/main/resources/META-INF/application.properties index 2db723927eaee10d39e02a6b2d369a06af7711fc..0fe4b240d97f087f00c28430740488f7e01f1577 100644 --- a/theodolite-benchmarks/uc3-beam/src/main/resources/META-INF/application.properties +++ b/theodolite-benchmarks/uc3-beam/src/main/resources/META-INF/application.properties @@ -17,6 +17,7 @@ num.threads=1 commit.interval.ms=1000 cache.max.bytes.buffering=-1 -specific.avro.reader=True -enable.auto.commit.config=True -auto.offset.reset.config=earliest \ No newline at end of file +specific.avro.reader=true +enable.auto.commit=true +max.poll.records=500 +auto.offset.reset=earliest \ No newline at end of file diff --git a/theodolite-benchmarks/uc4-beam/src/main/java/rocks/theodolite/benchmarks/uc4/beam/PipelineFactory.java b/theodolite-benchmarks/uc4-beam/src/main/java/rocks/theodolite/benchmarks/uc4/beam/PipelineFactory.java index 4cb707017ff90236df4546b87e472b86eb495e10..a71c24eda5385b10a73b9eb65a83bba8363dd3e7 100644 --- a/theodolite-benchmarks/uc4-beam/src/main/java/rocks/theodolite/benchmarks/uc4/beam/PipelineFactory.java +++ b/theodolite-benchmarks/uc4-beam/src/main/java/rocks/theodolite/benchmarks/uc4/beam/PipelineFactory.java @@ -251,10 +251,10 @@ public class PipelineFactory extends AbstractPipelineFactory { final Map<String, Object> consumerConfig = new HashMap<>(); consumerConfig.put( ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, - this.config.getString(ConfigurationKeys.ENABLE_AUTO_COMMIT_CONFIG)); + this.config.getString(ConfigurationKeys.ENABLE_AUTO_COMMIT)); consumerConfig.put( ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, - this.config.getString(ConfigurationKeys.AUTO_OFFSET_RESET_CONFIG)); + this.config.getString(ConfigurationKeys.AUTO_OFFSET_RESET)); consumerConfig.put( ConsumerConfig.GROUP_ID_CONFIG, this.config .getString(ConfigurationKeys.APPLICATION_NAME) + "-configuration"); @@ -265,10 +265,10 @@ public class PipelineFactory extends AbstractPipelineFactory { final Map<String, Object> consumerConfig = new HashMap<>(); consumerConfig.put( ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, - this.config.getString(ConfigurationKeys.ENABLE_AUTO_COMMIT_CONFIG)); + this.config.getString(ConfigurationKeys.ENABLE_AUTO_COMMIT)); consumerConfig.put( ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, - this.config.getString(ConfigurationKeys.AUTO_OFFSET_RESET_CONFIG)); + this.config.getString(ConfigurationKeys.AUTO_OFFSET_RESET)); consumerConfig.put( AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, this.config.getString(ConfigurationKeys.SCHEMA_REGISTRY_URL)); diff --git a/theodolite-benchmarks/uc4-beam/src/main/resources/META-INF/application.properties b/theodolite-benchmarks/uc4-beam/src/main/resources/META-INF/application.properties index bc679580dadf969e181b6787e8287066426be7e2..c1a8ca17b41ab8c8f0fa939c748200db5ba7d0d2 100644 --- a/theodolite-benchmarks/uc4-beam/src/main/resources/META-INF/application.properties +++ b/theodolite-benchmarks/uc4-beam/src/main/resources/META-INF/application.properties @@ -20,6 +20,7 @@ num.threads=1 commit.interval.ms=1000 cache.max.bytes.buffering=-1 -specific.avro.reader=True -enable.auto.commit.config=True -auto.offset.reset.config=earliest \ No newline at end of file +specific.avro.reader=true +enable.auto.commit=true +max.poll.records=500 +auto.offset.reset=earliest \ No newline at end of file diff --git a/theodolite/build.gradle b/theodolite/build.gradle index a066e94f09b71720f9392947640b077b153ccb9c..521137d7315de193f26fdf2307155e587c0dd921 100644 --- a/theodolite/build.gradle +++ b/theodolite/build.gradle @@ -59,6 +59,12 @@ compileTestKotlin { kotlinOptions.jvmTarget = JavaVersion.VERSION_11 } +test { + // Required because of https://github.com/quarkusio/quarkus/issues/18973 + minHeapSize = "256m" + maxHeapSize = "1024m" +} + detekt { failFast = true // fail build on any finding buildUponDefaultConfig = true diff --git a/theodolite/gradle.properties b/theodolite/gradle.properties index fd5768bc24a65dbd43b3ea770c854ae7c0da0a91..a7c5a39013639072e1ef47f9226b95b513d678d7 100644 --- a/theodolite/gradle.properties +++ b/theodolite/gradle.properties @@ -1,8 +1,8 @@ #Gradle properties quarkusPluginId=io.quarkus -quarkusPluginVersion=2.6.3.Final +quarkusPluginVersion=2.7.4.Final quarkusPlatformGroupId=io.quarkus.platform quarkusPlatformArtifactId=quarkus-bom -quarkusPlatformVersion=2.6.3.Final +quarkusPlatformVersion=2.7.4.Final #org.gradle.logging.level=INFO \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/ActionCommand.kt b/theodolite/src/main/kotlin/theodolite/benchmark/ActionCommand.kt index 966fa56329c8d7d466dd14858bcbc06bb5b857c3..a4345c43ac6a75667c3c3e85c8534697193e1458 100644 --- a/theodolite/src/main/kotlin/theodolite/benchmark/ActionCommand.kt +++ b/theodolite/src/main/kotlin/theodolite/benchmark/ActionCommand.kt @@ -7,7 +7,6 @@ import io.fabric8.kubernetes.client.dsl.ExecListener import io.fabric8.kubernetes.client.dsl.ExecWatch import io.fabric8.kubernetes.client.utils.Serialization import mu.KotlinLogging -import okhttp3.Response import theodolite.util.ActionCommandFailedException import theodolite.util.Configuration import java.io.ByteArrayOutputStream @@ -145,10 +144,8 @@ class ActionCommand(val client: NamespacedKubernetesClient) { } private class ActionCommandListener(val execLatch: CountDownLatch) : ExecListener { - override fun onOpen(response: Response) { - } - override fun onFailure(throwable: Throwable, response: Response) { + override fun onFailure(throwable: Throwable, response: ExecListener.Response) { execLatch.countDown() throw ActionCommandFailedException("Some error encountered while executing action, caused ${throwable.message})") } diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt b/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt index f85b83497e5d69e43c1d4784ef86170a5436e929..eea5b15cb1db7242328033a1bc46fb224d287bc2 100644 --- a/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt +++ b/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt @@ -1,23 +1,21 @@ package theodolite.benchmark import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.HasMetadata import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.client.KubernetesClientException import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.quarkus.runtime.annotations.RegisterForReflection -import theodolite.k8s.resourceLoader.K8sResourceLoaderFromString import theodolite.util.DeploymentFailedException -import theodolite.util.YamlParserFromString import java.lang.IllegalArgumentException @RegisterForReflection @JsonDeserialize class ConfigMapResourceSet : ResourceSet, KubernetesResource { lateinit var name: String - lateinit var files: List<String> // load all files, iff files is not set + var files: List<String>? = null // load all files, iff files is not set - override fun getResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, KubernetesResource>> { - val loader = K8sResourceLoaderFromString(client) + override fun getResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, HasMetadata>> { var resources: Map<String, String> try { @@ -31,9 +29,9 @@ class ConfigMapResourceSet : ResourceSet, KubernetesResource { throw DeploymentFailedException("Cannot find or read ConfigMap with name '$name'.", e) } - if (::files.isInitialized) { - val filteredResources = resources.filter { files.contains(it.key) } - if (filteredResources.size != files.size) { + files?.run { + val filteredResources = resources.filter { this.contains(it.key) } + if (filteredResources.size != this.size) { throw DeploymentFailedException("Could not find all specified Kubernetes manifests files") } resources = filteredResources @@ -43,14 +41,8 @@ class ConfigMapResourceSet : ResourceSet, KubernetesResource { resources .map { Pair( - getKind(resource = it.value), - it - ) - } - .map { - Pair( - it.second.key, - loader.loadK8sResource(it.first, it.second.value) + it.key, // filename + client.resource(it.value).get() ) } } catch (e: IllegalArgumentException) { @@ -59,11 +51,4 @@ class ConfigMapResourceSet : ResourceSet, KubernetesResource { } - private fun getKind(resource: String): String { - val parser = YamlParserFromString() - val resourceAsMap = parser.parse(resource, HashMap<String, String>()::class.java) - - return resourceAsMap?.get("kind") - ?: throw DeploymentFailedException("Could not find field kind of Kubernetes resource: ${resourceAsMap?.get("name")}") - } } \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/FileSystemResourceSet.kt b/theodolite/src/main/kotlin/theodolite/benchmark/FileSystemResourceSet.kt index f830232de4b6956fa0f989cae131903377862e6c..e95a637ab88f11902062de73b0c34603b08aded3 100644 --- a/theodolite/src/main/kotlin/theodolite/benchmark/FileSystemResourceSet.kt +++ b/theodolite/src/main/kotlin/theodolite/benchmark/FileSystemResourceSet.kt @@ -1,62 +1,57 @@ package theodolite.benchmark import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.HasMetadata import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.quarkus.runtime.annotations.RegisterForReflection -import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile import theodolite.util.DeploymentFailedException -import theodolite.util.YamlParserFromFile -import java.io.File +import java.io.BufferedReader +import java.io.FileInputStream import java.io.FileNotFoundException -import java.lang.IllegalArgumentException +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets +import java.nio.file.Path +import java.nio.file.Paths +import java.util.stream.Collectors +import kotlin.io.path.listDirectoryEntries + @RegisterForReflection @JsonDeserialize class FileSystemResourceSet: ResourceSet, KubernetesResource { lateinit var path: String - lateinit var files: List<String> - - override fun getResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, KubernetesResource>> { - - //if files is set ... - if(::files.isInitialized){ - return files - .map { loadSingleResource(resourceURL = it, client = client) } - } - - return try { - File(path) - .list() !! - .filter { it.endsWith(".yaml") || it.endsWith(".yml") } - .map { - loadSingleResource(resourceURL = it, client = client) - } - } catch (e: NullPointerException) { + var files: List<String>? = null + + override fun getResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, HasMetadata>> { + // if files is set ... + return files?.run { + return this + .map { Paths.get(path, it) } + .map { loadSingleResource(resource = it, client = client) } + } ?: + try { + Paths.get(path) + .listDirectoryEntries() + .filter { it.toString().endsWith(".yaml") || it.toString().endsWith(".yml") } + .map { loadSingleResource(resource = it, client = client) } + } catch (e: java.nio.file.NoSuchFileException) { // not to be confused with Kotlin exception throw DeploymentFailedException("Could not load files located in $path", e) } } - private fun loadSingleResource(resourceURL: String, client: NamespacedKubernetesClient): Pair<String, KubernetesResource> { - val parser = YamlParserFromFile() - val loader = K8sResourceLoaderFromFile(client) - val resourcePath = "$path/$resourceURL" - lateinit var kind: String - - try { - kind = parser.parse(resourcePath, HashMap<String, String>()::class.java)?.get("kind")!! - } catch (e: NullPointerException) { - throw DeploymentFailedException("Can not get Kind from resource $resourcePath", e) - } catch (e: FileNotFoundException){ - throw DeploymentFailedException("File $resourcePath not found", e) - - } - + private fun loadSingleResource(resource: Path, client: NamespacedKubernetesClient): Pair<String, HasMetadata> { return try { - val k8sResource = loader.loadK8sResource(kind, resourcePath) - Pair(resourceURL, k8sResource) + val stream = FileInputStream(resource.toFile()) + val text = BufferedReader( + InputStreamReader(stream, StandardCharsets.UTF_8) + ).lines().collect(Collectors.joining("\n")) + val k8sResource = client.resource(text).get() + Pair(resource.last().toString(), k8sResource) + } catch (e: FileNotFoundException){ + throw DeploymentFailedException("File $resource not found.", e) } catch (e: IllegalArgumentException) { - throw DeploymentFailedException("Could not load resource: $resourcePath", e) + throw DeploymentFailedException("Could not load resource: $resource.", e) } } } \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt index d42c2ea3c0ed5394fdcf5b89be0fe0470a15ba62..6857b9bf8918593dbe5085f40eb28fd8bd809d85 100644 --- a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt +++ b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmark.kt @@ -1,13 +1,13 @@ package theodolite.benchmark import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.HasMetadata import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.client.DefaultKubernetesClient import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.quarkus.runtime.annotations.RegisterForReflection import mu.KotlinLogging import theodolite.k8s.K8sManager -import theodolite.k8s.resourceLoader.K8sResourceLoader import theodolite.patcher.PatcherFactory import theodolite.util.* @@ -53,22 +53,27 @@ class KubernetesBenchmark : KubernetesResource, Benchmark { * It first loads them via the [YamlParserFromFile] to check for their concrete type and afterwards initializes them using * the [K8sResourceLoader] */ - fun loadKubernetesResources(resourceSet: List<ResourceSets>): Collection<Pair<String, KubernetesResource>> { + @Deprecated("Use `loadResourceSet` from `ResourceSets`") + fun loadKubernetesResources(resourceSet: List<ResourceSets>): Collection<Pair<String, HasMetadata>> { + return loadResources(resourceSet) + } + + private fun loadResources(resourceSet: List<ResourceSets>): Collection<Pair<String, HasMetadata>> { return resourceSet.flatMap { it.loadResourceSet(this.client) } } override fun setupInfrastructure() { this.infrastructure.beforeActions.forEach { it.exec(client = client) } val kubernetesManager = K8sManager(this.client) - loadKubernetesResources(this.infrastructure.resources) - .map{it.second} + loadResources(this.infrastructure.resources) + .map { it.second } .forEach { kubernetesManager.deploy(it) } } override fun teardownInfrastructure() { val kubernetesManager = K8sManager(this.client) - loadKubernetesResources(this.infrastructure.resources) - .map{it.second} + loadResources(this.infrastructure.resources) + .map { it.second } .forEach { kubernetesManager.remove(it) } this.infrastructure.afterActions.forEach { it.exec(client = client) } } @@ -91,8 +96,8 @@ class KubernetesBenchmark : KubernetesResource, Benchmark { ): BenchmarkDeployment { logger.info { "Using $namespace as namespace." } - val appResources = loadKubernetesResources(this.sut.resources) - val loadGenResources = loadKubernetesResources(this.loadGenerator.resources) + val appResources = loadResources(this.sut.resources) + val loadGenResources = loadResources(this.loadGenerator.resources) val patcherFactory = PatcherFactory() @@ -122,7 +127,7 @@ class KubernetesBenchmark : KubernetesResource, Benchmark { loadGenResources = loadGenResources.map { it.second }, loadGenerationDelay = loadGenerationDelay, afterTeardownDelay = afterTeardownDelay, - kafkaConfig = if (kafkaConfig != null) hashMapOf("bootstrap.servers" to kafkaConfig.bootstrapServer) else mapOf(), + kafkaConfig = if (kafkaConfig != null) mapOf("bootstrap.servers" to kafkaConfig.bootstrapServer) else mapOf(), topics = kafkaConfig?.topics ?: listOf(), client = this.client ) diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt index 2b3cf0fa13d894424e6a0546993e2fd9998b8620..b30032c524b1e421301e0e9d1ffe83772b43d900 100644 --- a/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt +++ b/theodolite/src/main/kotlin/theodolite/benchmark/KubernetesBenchmarkDeployment.kt @@ -1,5 +1,6 @@ package theodolite.benchmark +import io.fabric8.kubernetes.api.model.HasMetadata import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.quarkus.runtime.annotations.RegisterForReflection @@ -27,8 +28,8 @@ class KubernetesBenchmarkDeployment( private val sutAfterActions: List<Action>, private val loadGenBeforeActions: List<Action>, private val loadGenAfterActions: List<Action>, - val appResources: List<KubernetesResource>, - val loadGenResources: List<KubernetesResource>, + val appResources: List<HasMetadata>, + val loadGenResources: List<HasMetadata>, private val loadGenerationDelay: Long, private val afterTeardownDelay: Long, private val kafkaConfig: Map<String, Any>, @@ -79,7 +80,7 @@ class KubernetesBenchmarkDeployment( labelName = LAG_EXPORTER_POD_LABEL_NAME, labelValue = LAG_EXPORTER_POD_LABEL_VALUE ) - logger.info { "Teardown complete. Wait $afterTeardownDelay ms to let everything come down." } + logger.info { "Teardown complete. Wait $afterTeardownDelay seconds to let everything cool down." } Thread.sleep(Duration.ofSeconds(afterTeardownDelay).toMillis()) } } diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSets.kt b/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSets.kt index b6364949727d4ea134e348ce8b79e22334753c1c..0626a6e24369348d50b60fbb555665c58dd17281 100644 --- a/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSets.kt +++ b/theodolite/src/main/kotlin/theodolite/benchmark/ResourceSets.kt @@ -3,6 +3,7 @@ package theodolite.benchmark import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.databind.annotation.JsonDeserialize +import io.fabric8.kubernetes.api.model.HasMetadata import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.quarkus.runtime.annotations.RegisterForReflection @@ -19,13 +20,14 @@ class ResourceSets: KubernetesResource { @JsonInclude(JsonInclude.Include.NON_NULL) var fileSystem: FileSystemResourceSet? = null - fun loadResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, KubernetesResource>> { + fun loadResourceSet(client: NamespacedKubernetesClient): Collection<Pair<String, HasMetadata>> { + // TODO Find out whether field access (::configMap) is really what we want to do here (see #362) return if (::configMap != null) { - configMap?.getResourceSet(client= client) !! + configMap?.getResourceSet(client= client) !! } else if (::fileSystem != null) { - fileSystem?.getResourceSet(client= client ) !! + fileSystem?.getResourceSet(client= client ) !! } else { - throw DeploymentFailedException("could not load resourceSet.") + throw DeploymentFailedException("Could not load resourceSet.") } } } \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt index 93536282e2eefe6e476c3fde3fd86860fa24dcc3..84343ea7e8d7d420bcf320f36be02c39c41a1945 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/AbstractStateHandler.kt @@ -25,7 +25,12 @@ abstract class AbstractStateHandler<S : HasMetadata>( val resource = this.crdClient.withName(resourceName).get() if (resource != null) { val resourcePatched = setter(resource) - this.crdClient.patchStatus(resourcePatched) + // TODO replace with this.crdClient.replaceStatus(resourcePatched) with upcoming fabric8 release (> 5.12.1) + // find out the difference between patchStatus and replaceStatus + // see also https://github.com/fabric8io/kubernetes-client/pull/3798 + if (resourcePatched != null) { + this.crdClient.withName(resourcePatched.metadata.name).patchStatus(resourcePatched) + } } } catch (e: KubernetesClientException) { logger.warn(e) { "Status cannot be set for resource $resourceName." } diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt index 1ce94c2fdd1ce13d50a21e01b9d4692c87d0da6f..558d06ce03074c38741b6c0a72c6ffa6eff96019 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/LeaderElector.kt @@ -18,22 +18,22 @@ class LeaderElector( ) { // TODO(what is the name of the lock? .withName() or LeaseLock(..,name..) ?) - fun getLeadership(leader: KFunction0<Unit>) { + fun getLeadership(leader: () -> Unit) { val lockIdentity: String = UUID.randomUUID().toString() DefaultKubernetesClient().use { kc -> kc.leaderElector() .withConfig( LeaderElectionConfigBuilder() .withName("Theodolite") - .withLeaseDuration(Duration.ofSeconds(15L)) + .withLeaseDuration(Duration.ofSeconds(15)) .withLock(LeaseLock(client.namespace, name, lockIdentity)) - .withRenewDeadline(Duration.ofSeconds(10L)) - .withRetryPeriod(Duration.ofSeconds(2L)) + .withRenewDeadline(Duration.ofSeconds(10)) + .withRetryPeriod(Duration.ofSeconds(2)) .withLeaderCallbacks(LeaderCallbacks( { Thread { leader() }.start() }, - { logger.info { "STOPPED LEADERSHIP" } } + { logger.info { "Stop being the leading operator." } } ) { newLeader: String? -> - logger.info { "New leader elected $newLeader" } + logger.info { "New leader elected: $newLeader" } }) .build() ) diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt index 5f4180b0b4b58fa94b979c71998314baae63a91b..d9cb33b189da02b807301dde8550f2ae532d7e5a 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteController.kt @@ -130,8 +130,7 @@ class TheodoliteController( .list() .items .map { - it.spec.name = it.metadata.name - it + it.apply { it.spec.name = it.metadata.name } } } diff --git a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt index 071bd06071345499d01595df72e5de4c8535b3fc..ada30ec945dd602dabe3ddb5f0e635a4eeea7b5f 100644 --- a/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt +++ b/theodolite/src/main/kotlin/theodolite/execution/operator/TheodoliteOperator.kt @@ -41,15 +41,14 @@ class TheodoliteOperator { LeaderElector( client = client, name = Configuration.COMPONENT_NAME - ) - .getLeadership(::startOperator) + ).getLeadership(::startOperator) } /** * Start the operator. */ private fun startOperator() { - logger.info { "Using $namespace as namespace." } + logger.info { "Becoming the leading operator. Use namespace '$namespace'." } client.use { KubernetesDeserializer.registerCustomKind( "$GROUP/$API_VERSION", diff --git a/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt b/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt deleted file mode 100644 index 797ed88389947d66aa626ba2ef3fdf6732f8369d..0000000000000000000000000000000000000000 --- a/theodolite/src/main/kotlin/theodolite/k8s/CustomResourceWrapper.kt +++ /dev/null @@ -1,47 +0,0 @@ -package theodolite.k8s - -import io.fabric8.kubernetes.api.model.KubernetesResource -import io.fabric8.kubernetes.client.NamespacedKubernetesClient -import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext -import mu.KotlinLogging - -private val logger = KotlinLogging.logger {} - -class CustomResourceWrapper( - val crAsMap: Map<String, String>, - private val context: CustomResourceDefinitionContext -) : KubernetesResource { - /** - * Deploy a service monitor - * - * @param client a namespaced Kubernetes client which are used to deploy the CR object. - * - * @throws java.io.IOException if the resource could not be deployed. - */ - fun deploy(client: NamespacedKubernetesClient) { - client.customResource(this.context) - .createOrReplace(client.configuration.namespace, this.crAsMap as Map<String, Any>) - } - - /** - * Delete a service monitor - * - * @param client a namespaced Kubernetes client which are used to delete the CR object. - */ - fun delete(client: NamespacedKubernetesClient) { - try { - client.customResource(this.context) - .delete(client.configuration.namespace, this.getName()) - } catch (e: Exception) { - logger.warn { "Could not delete custom resource" } - } - } - - /** - * @throws NullPointerException if name or metadata is null - */ - fun getName(): String { - val metadataAsMap = this.crAsMap["metadata"]!! as Map<String, String> - return metadataAsMap["name"]!! - } -} diff --git a/theodolite/src/main/kotlin/theodolite/k8s/K8sContextFactory.kt b/theodolite/src/main/kotlin/theodolite/k8s/K8sContextFactory.kt index 7eb209bfbab02bb94d34c985aa308173e509d4e4..38224f26a38a241e92b38e8b92a7fa5b4e198f5e 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/K8sContextFactory.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/K8sContextFactory.kt @@ -7,6 +7,7 @@ import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext * * @see CustomResourceDefinitionContext */ +@Deprecated("Use `CustomResourceDefinitionContext.Builder` instead.") class K8sContextFactory { /** diff --git a/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt b/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt index 389d5eefad556df502c218862e2f253ef8ad2100..5b4880b45db76d9e68e87fda0ece5b04966439c8 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/K8sManager.kt @@ -1,6 +1,7 @@ package theodolite.k8s import io.fabric8.kubernetes.api.model.ConfigMap +import io.fabric8.kubernetes.api.model.HasMetadata import io.fabric8.kubernetes.api.model.KubernetesResource import io.fabric8.kubernetes.api.model.Service import io.fabric8.kubernetes.api.model.apps.Deployment @@ -21,49 +22,31 @@ class K8sManager(private val client: NamespacedKubernetesClient) { * Deploys different k8s resources using the client. * @throws IllegalArgumentException if KubernetesResource not supported. */ - fun deploy(resource: KubernetesResource) { - when (resource) { - is Deployment -> - this.client.apps().deployments().createOrReplace(resource) - is Service -> - this.client.services().createOrReplace(resource) - is ConfigMap -> - this.client.configMaps().createOrReplace(resource) - is StatefulSet -> - this.client.apps().statefulSets().createOrReplace(resource) - is CustomResourceWrapper -> resource.deploy(client) - else -> throw IllegalArgumentException("Unknown Kubernetes resource.") - } + fun deploy(resource: HasMetadata) { + client.resource(resource).createOrReplace() } /** * Removes different k8s resources using the client. * @throws IllegalArgumentException if KubernetesResource not supported. */ - fun remove(resource: KubernetesResource) { + fun remove(resource: HasMetadata) { + client.resource(resource).delete() when (resource) { is Deployment -> { - this.client.apps().deployments().delete(resource) ResourceByLabelHandler(client = client) .blockUntilPodsDeleted( matchLabels = resource.spec.selector.matchLabels ) logger.info { "Deployment '${resource.metadata.name}' deleted." } } - is Service -> - this.client.services().delete(resource) - is ConfigMap -> - this.client.configMaps().delete(resource) is StatefulSet -> { - this.client.apps().statefulSets().delete(resource) ResourceByLabelHandler(client = client) .blockUntilPodsDeleted( matchLabels = resource.spec.selector.matchLabels ) logger.info { "StatefulSet '$resource.metadata.name' deleted." } } - is CustomResourceWrapper -> resource.delete(client) - else -> throw IllegalArgumentException("Unknown Kubernetes resource.") } } } diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt deleted file mode 100644 index 36cfef9ce912886a638c200b502923dfe03ef5d0..0000000000000000000000000000000000000000 --- a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt +++ /dev/null @@ -1,84 +0,0 @@ -package theodolite.k8s.resourceLoader - -import io.fabric8.kubernetes.api.model.KubernetesResource -import mu.KotlinLogging -import theodolite.k8s.K8sContextFactory - -private val logger = KotlinLogging.logger {} - -abstract class AbstractK8sLoader: K8sResourceLoader { - - fun loadK8sResource(kind: String, resourceString: String): KubernetesResource { - return when (kind.replaceFirst(kind[0],kind[0].uppercaseChar())) { - "Deployment" -> loadDeployment(resourceString) - "Service" -> loadService(resourceString) - "ServiceMonitor" -> loadServiceMonitor(resourceString) - "PodMonitor" -> loadPodMonitor(resourceString) - "ConfigMap" -> loadConfigmap(resourceString) - "StatefulSet" -> loadStatefulSet(resourceString) - "Execution" -> loadExecution(resourceString) - "Benchmark" -> loadBenchmark(resourceString) - else -> { - logger.error { "Error during loading of unspecified resource Kind '$kind'." } - throw IllegalArgumentException("error while loading resource with kind: $kind") - } - } - } - - fun <T : KubernetesResource> loadGenericResource(resourceString: String, f: (String) -> T): T { - var resource: T? = null - - try { - resource = f(resourceString) - } catch (e: Exception) { - logger.warn { e } - } - - if (resource == null) { - throw IllegalArgumentException("The Resource: $resourceString could not be loaded") - } - return resource - } - - - - override fun loadServiceMonitor(resource: String): KubernetesResource { - val context = K8sContextFactory().create( - api = "v1", - scope = "Namespaced", - group = "monitoring.coreos.com", - plural = "servicemonitors" - ) - return loadCustomResourceWrapper(resource, context) - } - - override fun loadPodMonitor(resource: String): KubernetesResource { - val context = K8sContextFactory().create( - api = "v1", - scope = "Namespaced", - group = "monitoring.coreos.com", - plural = "podmonitors" - ) - return loadCustomResourceWrapper(resource, context) - } - - override fun loadExecution(resource: String): KubernetesResource { - val context = K8sContextFactory().create( - api = "v1", - scope = "Namespaced", - group = "theodolite.com", - plural = "executions" - ) - return loadCustomResourceWrapper(resource, context) - } - - override fun loadBenchmark(resource: String): KubernetesResource { - val context = K8sContextFactory().create( - api = "v1", - scope = "Namespaced", - group = "theodolite.com", - plural = "benchmarks" - ) - return loadCustomResourceWrapper(resource, context) - } -} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt deleted file mode 100644 index 1487b64bf4f7fbcc735539a429be9237d41205bc..0000000000000000000000000000000000000000 --- a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt +++ /dev/null @@ -1,16 +0,0 @@ -package theodolite.k8s.resourceLoader - -import io.fabric8.kubernetes.api.model.KubernetesResource -import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext - -interface K8sResourceLoader { - fun loadDeployment(resource: String): KubernetesResource - fun loadService(resource: String): KubernetesResource - fun loadStatefulSet(resource: String): KubernetesResource - fun loadExecution(resource: String): KubernetesResource - fun loadBenchmark(resource: String): KubernetesResource - fun loadConfigmap(resource: String): KubernetesResource - fun loadServiceMonitor(resource: String): KubernetesResource - fun loadPodMonitor(resource: String): KubernetesResource - fun loadCustomResourceWrapper(resource: String, context: CustomResourceDefinitionContext): KubernetesResource -} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromFile.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromFile.kt deleted file mode 100644 index 08f34e1d67c9821c9f9a07a49f4ba8683a072611..0000000000000000000000000000000000000000 --- a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromFile.kt +++ /dev/null @@ -1,75 +0,0 @@ -package theodolite.k8s.resourceLoader - -import io.fabric8.kubernetes.api.model.ConfigMap -import io.fabric8.kubernetes.api.model.KubernetesResource -import io.fabric8.kubernetes.api.model.Service -import io.fabric8.kubernetes.api.model.apps.Deployment -import io.fabric8.kubernetes.client.NamespacedKubernetesClient -import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext -import theodolite.k8s.CustomResourceWrapper -import theodolite.util.YamlParserFromFile - -/** - * Used to load different Kubernetes resources. - * Supports: Deployments, Services, ConfigMaps, and CustomResources. - * @param client KubernetesClient used to deploy or remove. - */ -class K8sResourceLoaderFromFile(private val client: NamespacedKubernetesClient): AbstractK8sLoader(), - K8sResourceLoader { - - /** - * Parses a Service from a service yaml - * @param resource of the yaml file - * @return Service from fabric8 - */ - override fun loadService(resource: String): Service { - return loadGenericResource(resource) { x: String -> client.services().load(x).get() } - } - - - /** - * Parses a CustomResource from a yaml - * @param path of the yaml file - * @param context specific crd context for this custom resource - * @return CustomResourceWrapper from fabric8 - */ - override fun loadCustomResourceWrapper(resource: String, context: CustomResourceDefinitionContext): CustomResourceWrapper { - return loadGenericResource(resource) { - CustomResourceWrapper( - YamlParserFromFile().parse( - resource, - HashMap<String, String>()::class.java - )!!, - context - ) - } - } - - /** - * Parses a Deployment from a Deployment yaml - * @param resource of the yaml file - * @return Deployment from fabric8 - */ - override fun loadDeployment(resource: String): Deployment { - return loadGenericResource(resource) { x: String -> client.apps().deployments().load(x).get() } - } - - /** - * Parses a ConfigMap from a ConfigMap yaml - * @param resource of the yaml file - * @return ConfigMap from fabric8 - */ - override fun loadConfigmap(resource: String): ConfigMap { - return loadGenericResource(resource) { x: String -> client.configMaps().load(x).get() } - } - - /** - * Parses a StatefulSet from a StatefulSet yaml - * @param resource of the yaml file - * @return StatefulSet from fabric8 - */ - override fun loadStatefulSet(resource: String): KubernetesResource { - return loadGenericResource(resource) { x: String -> client.apps().statefulSets().load(x).get() } - - } -} diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromString.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromString.kt deleted file mode 100644 index 639e4c4584d47968cd718d601f1cd7064d85eda2..0000000000000000000000000000000000000000 --- a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoaderFromString.kt +++ /dev/null @@ -1,55 +0,0 @@ -package theodolite.k8s.resourceLoader - -import io.fabric8.kubernetes.api.model.ConfigMap -import io.fabric8.kubernetes.api.model.KubernetesResource -import io.fabric8.kubernetes.api.model.Service -import io.fabric8.kubernetes.api.model.apps.Deployment -import io.fabric8.kubernetes.api.model.apps.StatefulSet -import io.fabric8.kubernetes.client.NamespacedKubernetesClient -import io.fabric8.kubernetes.client.dsl.base.CustomResourceDefinitionContext -import theodolite.k8s.CustomResourceWrapper -import theodolite.util.YamlParserFromString -import java.io.ByteArrayInputStream -import java.io.InputStream - -class K8sResourceLoaderFromString(private val client: NamespacedKubernetesClient): AbstractK8sLoader(), - K8sResourceLoader { - - override fun loadService(resource: String): Service { - return loadAnyResource(resource) { stream -> client.services().load(stream).get() } - } - - override fun loadDeployment(resource: String): Deployment { - return loadAnyResource(resource) { stream -> client.apps().deployments().load(stream).get() } - } - - override fun loadConfigmap(resource: String): ConfigMap { - return loadAnyResource(resource) { stream -> client.configMaps().load(stream).get() } - } - - override fun loadStatefulSet(resource: String): StatefulSet { - return loadAnyResource(resource) { stream -> client.apps().statefulSets().load(stream).get() } - } - - private fun <T : KubernetesResource> loadAnyResource(resource: String, f: (InputStream) -> T): T { - return loadGenericResource(resource) { f(ByteArrayInputStream(it.encodeToByteArray())) } - } - - /** - * Parses a CustomResource from a yaml - * @param resource of the yaml file - * @param context specific crd context for this custom resource - * @return CustomResourceWrapper from fabric8 - */ - override fun loadCustomResourceWrapper(resource: String, context: CustomResourceDefinitionContext): CustomResourceWrapper { - return loadGenericResource(resource) { - CustomResourceWrapper( - YamlParserFromString().parse( - resource, - HashMap<String, String>()::class.java - )!!, - context - ) - } - } -} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/util/PatcherDefinition.kt b/theodolite/src/main/kotlin/theodolite/util/PatcherDefinition.kt index 6ec0cce36751ec0343d40aa49fefa44f4c7fc918..fd2ac209a52e0d516ffa9ec07e465fa076ae665a 100644 --- a/theodolite/src/main/kotlin/theodolite/util/PatcherDefinition.kt +++ b/theodolite/src/main/kotlin/theodolite/util/PatcherDefinition.kt @@ -21,5 +21,5 @@ class PatcherDefinition { lateinit var resource: String @JsonSerialize - lateinit var properties: MutableMap<String, String> + lateinit var properties: Map<String, String> } diff --git a/theodolite/src/main/kotlin/theodolite/util/YamlParserFromFile.kt b/theodolite/src/main/kotlin/theodolite/util/YamlParserFromFile.kt index ae36349e628621bb7ad287d8cf557fbefa3ff5c5..58ca925e6aeeaca4f2f35c97c027ee2d24188e50 100644 --- a/theodolite/src/main/kotlin/theodolite/util/YamlParserFromFile.kt +++ b/theodolite/src/main/kotlin/theodolite/util/YamlParserFromFile.kt @@ -9,6 +9,7 @@ import java.io.InputStream /** * The YamlParser parses a YAML file */ +@Deprecated("Use Jackson ObjectMapper instead") class YamlParserFromFile : Parser { override fun <T> parse(path: String, E: Class<T>): T? { val input: InputStream = FileInputStream(File(path)) diff --git a/theodolite/src/main/kotlin/theodolite/util/YamlParserFromString.kt b/theodolite/src/main/kotlin/theodolite/util/YamlParserFromString.kt index 0e197908a501c0f6b89761a61989580b18e21f64..99c81f1ed674b2aa21f6aec7b3e0dff1b8c86840 100644 --- a/theodolite/src/main/kotlin/theodolite/util/YamlParserFromString.kt +++ b/theodolite/src/main/kotlin/theodolite/util/YamlParserFromString.kt @@ -6,6 +6,7 @@ import org.yaml.snakeyaml.constructor.Constructor /** * The YamlParser parses a YAML string */ +@Deprecated("Use Jackson ObjectMapper instead") class YamlParserFromString : Parser { override fun <T> parse(fileString: String, E: Class<T>): T? { val parser = Yaml(Constructor(E)) diff --git a/theodolite/src/test/kotlin/MockServerUtils.kt b/theodolite/src/test/kotlin/MockServerUtils.kt new file mode 100644 index 0000000000000000000000000000000000000000..0c8da45cd5d2dab948f9ccba1c2f4917050bb040 --- /dev/null +++ b/theodolite/src/test/kotlin/MockServerUtils.kt @@ -0,0 +1,20 @@ +import io.fabric8.kubernetes.api.model.APIResourceListBuilder +import io.fabric8.kubernetes.client.dsl.base.ResourceDefinitionContext +import io.fabric8.kubernetes.client.server.mock.KubernetesServer + +fun KubernetesServer.registerResource(context: ResourceDefinitionContext) { + val apiResourceList = APIResourceListBuilder() + .addNewResource() + .withName(context.plural) + .withKind(context.kind) + .withNamespaced(context.isNamespaceScoped) + .endResource() + .build() + + this + .expect() + .get() + .withPath("/apis/${context.group}/${context.version}") + .andReturn(200, apiResourceList) + .always() +} \ No newline at end of file diff --git a/theodolite/src/test/kotlin/theodolite/benchmark/ConfigMapResourceSetTest.kt b/theodolite/src/test/kotlin/theodolite/benchmark/ConfigMapResourceSetTest.kt index bc3263aa5fd06a8a19609d9f677db51f173cf54f..33a4572e368655744185312ff2352b1294d7bef6 100644 --- a/theodolite/src/test/kotlin/theodolite/benchmark/ConfigMapResourceSetTest.kt +++ b/theodolite/src/test/kotlin/theodolite/benchmark/ConfigMapResourceSetTest.kt @@ -1,32 +1,67 @@ package theodolite.benchmark -import com.google.gson.Gson +import com.fasterxml.jackson.databind.ObjectMapper import io.fabric8.kubernetes.api.model.* import io.fabric8.kubernetes.api.model.apps.Deployment import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder import io.fabric8.kubernetes.api.model.apps.StatefulSet import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder +import io.fabric8.kubernetes.client.dsl.MixedOperation +import io.fabric8.kubernetes.client.dsl.Resource +import io.fabric8.kubernetes.client.dsl.base.ResourceDefinitionContext import io.fabric8.kubernetes.client.server.mock.KubernetesServer import io.quarkus.test.junit.QuarkusTest +import io.quarkus.test.kubernetes.client.KubernetesTestServer +import io.quarkus.test.kubernetes.client.WithKubernetesTestServer import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows -import theodolite.k8s.CustomResourceWrapper -import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile +import org.mockito.kotlin.mock +import registerResource +import theodolite.TestBenchmark +import theodolite.execution.operator.BenchmarkCRDummy +import theodolite.execution.operator.ExecutionClient +import theodolite.execution.operator.ExecutionEventHandler +import theodolite.execution.operator.ExecutionStateHandler +import theodolite.model.crd.BenchmarkCRD +import theodolite.model.crd.ExecutionCRD import theodolite.util.DeploymentFailedException +import java.io.FileInputStream -private const val testResourcePath = "./src/test/resources/k8s-resource-files/" +// TODO move somewhere else +typealias BenchmarkClient = MixedOperation<BenchmarkCRD, KubernetesResourceList<BenchmarkCRD>, Resource<BenchmarkCRD>> @QuarkusTest -class ConfigMapResourceSetTest { - private val server = KubernetesServer(false, true) +@WithKubernetesTestServer +internal class ConfigMapResourceSetTest { + + @KubernetesTestServer + private lateinit var server: KubernetesServer + + private val objectMapper: ObjectMapper = ObjectMapper() + + private lateinit var executionClient: ExecutionClient + private lateinit var benchmarkClient: BenchmarkClient @BeforeEach fun setUp() { server.before() + this.server.client + .apiextensions().v1() + .customResourceDefinitions() + .load(FileInputStream("crd/crd-execution.yaml")) + .create() + this.server.client + .apiextensions().v1() + .customResourceDefinitions() + .load(FileInputStream("crd/crd-benchmark.yaml")) + .create() + + this.executionClient = this.server.client.resources(ExecutionCRD::class.java) + this.benchmarkClient = this.server.client.resources(BenchmarkCRD::class.java) } @AfterEach @@ -34,184 +69,196 @@ class ConfigMapResourceSetTest { server.after() } - fun deployAndGetResource(resource: String): Collection<Pair<String, KubernetesResource>> { - val configMap1 = ConfigMapBuilder() + private fun deployAndGetResource(vararg resources: HasMetadata): ConfigMapResourceSet { + val configMap = ConfigMapBuilder() .withNewMetadata().withName("test-configmap").endMetadata() - .addToData("test-resource.yaml",resource) + .let { + resources.foldIndexed(it) { + i, b, r -> b.addToData("resource_$i.yaml", objectMapper.writeValueAsString(r)) + } + } .build() - server.client.configMaps().createOrReplace(configMap1) + server.client.configMaps().createOrReplace(configMap) val resourceSet = ConfigMapResourceSet() resourceSet.name = "test-configmap" - return resourceSet.getResourceSet(server.client) + return resourceSet } - @Test fun testLoadDeployment() { - val resourceBuilder = DeploymentBuilder() - resourceBuilder.withNewSpec().endSpec() - resourceBuilder.withNewMetadata().endMetadata() - val resource = resourceBuilder.build() - resource.metadata.name = "test-deployment" + val resource = DeploymentBuilder() + .withNewSpec() + .endSpec() + .withNewMetadata() + .withName("test-deployment") + .endMetadata() + .build() - val createdResource = deployAndGetResource(resource = Gson().toJson(resource)) + val createdResource = deployAndGetResource(resource).getResourceSet(server.client) assertEquals(1, createdResource.size) - assertTrue(createdResource.toMutableSet().first().second is Deployment) - assertTrue(createdResource.toMutableSet().first().second.toString().contains(other = resource.metadata.name)) + assertTrue(createdResource.toList().first().second is Deployment) + assertTrue(createdResource.toList().first().second.toString().contains(other = resource.metadata.name)) } @Test fun testLoadStateFulSet() { - val resourceBuilder = StatefulSetBuilder() - resourceBuilder.withNewSpec().endSpec() - resourceBuilder.withNewMetadata().endMetadata() - val resource = resourceBuilder.build() - resource.metadata.name = "test-resource" + val resource = StatefulSetBuilder() + .withNewSpec() + .endSpec() + .withNewMetadata() + .withName("test-sts") + .endMetadata() + .build() - val createdResource = deployAndGetResource(resource = Gson().toJson(resource)) + val createdResource = deployAndGetResource(resource).getResourceSet(server.client) assertEquals(1, createdResource.size) - assertTrue(createdResource.toMutableSet().first().second is StatefulSet) - assertTrue(createdResource.toMutableSet().first().second.toString().contains(other = resource.metadata.name)) + assertTrue(createdResource.toList().first().second is StatefulSet) + assertTrue(createdResource.toList().first().second.toString().contains(other = resource.metadata.name)) } @Test fun testLoadService() { - val resourceBuilder = ServiceBuilder() - resourceBuilder.withNewSpec().endSpec() - resourceBuilder.withNewMetadata().endMetadata() - val resource = resourceBuilder.build() - resource.metadata.name = "test-resource" + val resource = ServiceBuilder() + .withNewSpec() + .endSpec() + .withNewMetadata() + .withName("test-service") + .endMetadata() + .build() - val createdResource = deployAndGetResource(resource = Gson().toJson(resource)) + val createdResource = deployAndGetResource(resource).getResourceSet(server.client) assertEquals(1, createdResource.size) - assertTrue(createdResource.toMutableSet().first().second is Service) - assertTrue(createdResource.toMutableSet().first().second.toString().contains(other = resource.metadata.name)) + assertTrue(createdResource.toList().first().second is Service) + assertTrue(createdResource.toList().first().second.toString().contains(other = resource.metadata.name)) } @Test fun testLoadConfigMap() { - val resourceBuilder = ConfigMapBuilder() - resourceBuilder.withNewMetadata().endMetadata() - val resource = resourceBuilder.build() - resource.metadata.name = "test-resource" + val resource = ConfigMapBuilder() + .withNewMetadata() + .withName("test-configmap") + .endMetadata() + .build() - val createdResource = deployAndGetResource(resource = Gson().toJson(resource)) + val createdResource = deployAndGetResource(resource).getResourceSet(server.client) assertEquals(1, createdResource.size) - assertTrue(createdResource.toMutableSet().first().second is ConfigMap) - assertTrue(createdResource.toMutableSet().first().second.toString().contains(other = resource.metadata.name)) + assertTrue(createdResource.toList().first().second is ConfigMap) + assertTrue(createdResource.toList().first().second.toString().contains(other = resource.metadata.name)) } @Test fun testLoadExecution() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("Execution", testResourcePath + "test-execution.yaml") as CustomResourceWrapper - val createdResource = deployAndGetResource(resource = Gson().toJson(resource.crAsMap)) + val stream = javaClass.getResourceAsStream("/k8s-resource-files/test-execution.yaml") + val execution = this.executionClient.load(stream).get() + val createdResource = deployAndGetResource(execution).getResourceSet(server.client) assertEquals(1, createdResource.size) - assertTrue(createdResource.toMutableSet().first().second is CustomResourceWrapper) + val loadedResource = createdResource.toList().first().second + assertTrue(loadedResource is ExecutionCRD) + assertEquals("example-execution", loadedResource.metadata.name) + - val loadedResource = createdResource.toMutableSet().first().second - if (loadedResource is CustomResourceWrapper){ - assertTrue(loadedResource.getName() == "example-execution") - } } @Test fun testLoadBenchmark() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("Benchmark", testResourcePath + "test-benchmark.yaml") as CustomResourceWrapper - val createdResource = deployAndGetResource(resource = Gson().toJson(resource.crAsMap)) + val benchmark = BenchmarkCRDummy("example-benchmark").getCR() + val createdResource = deployAndGetResource(benchmark).getResourceSet(server.client) assertEquals(1, createdResource.size) - assertTrue(createdResource.toMutableSet().first().second is CustomResourceWrapper) - - val loadedResource = createdResource.toMutableSet().first().second - if (loadedResource is CustomResourceWrapper){ - assertTrue(loadedResource.getName() == "example-benchmark") - } + val loadedResource = createdResource.toList().first().second + assertTrue(loadedResource is BenchmarkCRD) + assertEquals("example-benchmark", loadedResource.metadata.name) } @Test fun testLoadServiceMonitor() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("ServiceMonitor", testResourcePath + "test-service-monitor.yaml") as CustomResourceWrapper - val createdResource = deployAndGetResource(resource = Gson().toJson(resource.crAsMap)) + val serviceMonitorContext = ResourceDefinitionContext.Builder() + .withGroup("monitoring.coreos.com") + .withKind("ServiceMonitor") + .withPlural("servicemonitors") + .withNamespaced(true) + .withVersion("v1") + .build() + server.registerResource(serviceMonitorContext) - assertEquals(1, createdResource.size) - assertTrue(createdResource.toMutableSet().first().second is CustomResourceWrapper) + val stream = javaClass.getResourceAsStream("/k8s-resource-files/test-service-monitor.yaml") + val serviceMonitor = server.client.load(stream).get()[0] + val createdResource = deployAndGetResource(serviceMonitor).getResourceSet(server.client) - val loadedResource = createdResource.toMutableSet().first().second - if (loadedResource is CustomResourceWrapper){ - assertTrue(loadedResource.getName() == "test-service-monitor") - } + assertEquals(1, createdResource.size) + val loadedResource = createdResource.toList().first().second + assertTrue(loadedResource is GenericKubernetesResource) + assertEquals("ServiceMonitor", loadedResource.kind) + assertEquals("test-service-monitor", loadedResource.metadata.name) } @Test fun testMultipleFiles(){ - val resourceBuilder = DeploymentBuilder() - resourceBuilder.withNewSpec().endSpec() - resourceBuilder.withNewMetadata().endMetadata() - val resource = resourceBuilder.build() - resource.metadata.name = "test-deployment" - - val resourceBuilder1 = ConfigMapBuilder() - resourceBuilder1.withNewMetadata().endMetadata() - val resource1 = resourceBuilder1.build() - resource1.metadata.name = "test-configmap" - - val configMap1 = ConfigMapBuilder() - .withNewMetadata().withName("test-configmap").endMetadata() - .addToData("test-deployment.yaml",Gson().toJson(resource)) - .addToData("test-configmap.yaml",Gson().toJson(resource1)) + val deployment = DeploymentBuilder() + .withNewSpec() + .endSpec() + .withNewMetadata() + .withName("test-deployment") + .endMetadata() + .build() + val configMap = ConfigMapBuilder() + .withNewMetadata() + .withName("test-configmap") + .endMetadata() .build() - server.client.configMaps().createOrReplace(configMap1) - - val resourceSet = ConfigMapResourceSet() - resourceSet.name = "test-configmap" - - val createdResourcesSet = resourceSet.getResourceSet(server.client) + val createdResourceSet = deployAndGetResource(deployment, configMap).getResourceSet(server.client) - assertEquals(2,createdResourcesSet.size ) - assert(createdResourcesSet.toMutableList()[0].second is Deployment) - assert(createdResourcesSet.toMutableList()[1].second is ConfigMap) + assertEquals(2, createdResourceSet.size ) + assert(createdResourceSet.toList()[0].second is Deployment) + assert(createdResourceSet.toList()[1].second is ConfigMap) } @Test - fun testFileIsSet(){ - val resourceBuilder = DeploymentBuilder() - resourceBuilder.withNewSpec().endSpec() - resourceBuilder.withNewMetadata().endMetadata() - val resource = resourceBuilder.build() - resource.metadata.name = "test-deployment" - - val resourceBuilder1 = ConfigMapBuilder() - resourceBuilder1.withNewMetadata().endMetadata() - val resource1 = resourceBuilder1.build() - resource1.metadata.name = "test-configmap" - - val configMap1 = ConfigMapBuilder() - .withNewMetadata().withName("test-configmap").endMetadata() - .addToData("test-deployment.yaml",Gson().toJson(resource)) - .addToData("test-configmap.yaml",Gson().toJson(resource1)) + fun testFilesRestricted() { + val deployment = DeploymentBuilder() + .withNewSpec() + .endSpec() + .withNewMetadata() + .withName("test-deployment") + .endMetadata() + .build() + val configMap = ConfigMapBuilder() + .withNewMetadata() + .withName("test-configmap") + .endMetadata() .build() - server.client.configMaps().createOrReplace(configMap1) - - val resourceSet = ConfigMapResourceSet() - resourceSet.name = "test-configmap" - resourceSet.files = listOf("test-deployment.yaml") + val createdResourceSet = deployAndGetResource(deployment, configMap) + val allResources = createdResourceSet.getResourceSet(server.client) + assertEquals(2, allResources.size) + createdResourceSet.files = listOf(allResources.first().first) // only select first file from ConfigMa + val resources = createdResourceSet.getResourceSet(server.client) + assertEquals(1, resources.size) + assertTrue(resources.toList().first().second is Deployment) + } - val createdResourcesSet = resourceSet.getResourceSet(server.client) + @Test + fun testFileNotExist() { + val resource = DeploymentBuilder() + .withNewSpec() + .endSpec() + .withNewMetadata() + .withName("test-deployment") + .endMetadata() + .build() - assertEquals(1, createdResourcesSet.size ) - assert(createdResourcesSet.toMutableSet().first().second is Deployment) + val resourceSet = deployAndGetResource(resource) + resourceSet.files = listOf("non-existing-file.yaml") + assertThrows<DeploymentFailedException> { + resourceSet.getResourceSet(server.client) + } } - @Test fun testConfigMapNotExist() { val resourceSet = ConfigMapResourceSet() diff --git a/theodolite/src/test/kotlin/theodolite/benchmark/FileSystemResourceSetTest.kt b/theodolite/src/test/kotlin/theodolite/benchmark/FileSystemResourceSetTest.kt index f15685c8e0ecd67b99caabb77f68cc35a78b47f2..6a31875d00c8f578dcc475c3de21e130c595f673 100644 --- a/theodolite/src/test/kotlin/theodolite/benchmark/FileSystemResourceSetTest.kt +++ b/theodolite/src/test/kotlin/theodolite/benchmark/FileSystemResourceSetTest.kt @@ -1,28 +1,53 @@ package theodolite.benchmark -import io.fabric8.kubernetes.api.model.ConfigMap -import io.fabric8.kubernetes.api.model.Service +import io.fabric8.kubernetes.api.model.* import io.fabric8.kubernetes.api.model.apps.Deployment import io.fabric8.kubernetes.api.model.apps.StatefulSet +import io.fabric8.kubernetes.client.dsl.base.ResourceDefinitionContext import io.fabric8.kubernetes.client.server.mock.KubernetesServer -import org.junit.jupiter.api.AfterEach +import io.quarkus.test.junit.QuarkusTest +import io.quarkus.test.kubernetes.client.KubernetesTestServer +import io.quarkus.test.kubernetes.client.WithKubernetesTestServer +import org.junit.jupiter.api.* import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows -import theodolite.k8s.CustomResourceWrapper +import org.junit.jupiter.api.io.TempDir +import registerResource +import theodolite.model.crd.BenchmarkCRD +import theodolite.model.crd.ExecutionCRD import theodolite.util.DeploymentFailedException +import java.io.FileInputStream +import java.nio.file.Files +import java.nio.file.Path -private const val testResourcePath = "./src/test/resources/k8s-resource-files/" - +@QuarkusTest +@WithKubernetesTestServer class FileSystemResourceSetTest { - private val server = KubernetesServer(false, true) + @KubernetesTestServer + private lateinit var server: KubernetesServer + + @TempDir + @JvmField + final var tempDir: Path? = null @BeforeEach fun setUp() { server.before() + this.server.client + .apiextensions().v1() + .customResourceDefinitions() + .load(FileInputStream("crd/crd-execution.yaml")) + .create() + this.server.client + .apiextensions().v1() + .customResourceDefinitions() + .load(FileInputStream("crd/crd-benchmark.yaml")) + .create() + + // Apparently we need create CRD clients once + this.server.client.resources(ExecutionCRD::class.java) + this.server.client.resources(BenchmarkCRD::class.java) } @AfterEach @@ -30,80 +55,113 @@ class FileSystemResourceSetTest { server.after() } + private fun copyTestResourceFile(fileName: String, tempDir: Path) { + val stream = javaClass.getResourceAsStream("/k8s-resource-files/$fileName") + ?: throw IllegalArgumentException("File does not exist") + val target = tempDir.resolve(fileName) + Files.copy(stream, target) + } + @Test - fun testLoadDeployment() { + fun testLoadDeployment(@TempDir tempDir: Path) { + copyTestResourceFile("test-deployment.yaml", tempDir) + val resourceSet = FileSystemResourceSet() - resourceSet.path = testResourcePath + resourceSet.path = tempDir.toString() resourceSet.files = listOf("test-deployment.yaml") - assertEquals(1,resourceSet.getResourceSet(server.client).size) - assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is Deployment) + assertEquals(1, resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toList().first().second is Deployment) } @Test - fun testLoadService() { + fun testLoadService(@TempDir tempDir: Path) { + copyTestResourceFile("test-service.yaml", tempDir) + val resourceSet = FileSystemResourceSet() - resourceSet.path = testResourcePath + resourceSet.path = tempDir.toString() resourceSet.files = listOf("test-service.yaml") - assertEquals(1,resourceSet.getResourceSet(server.client).size) - assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is Service) + assertEquals(1, resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toList().first().second is Service) } @Test - fun testLoadStatefulSet() { + fun testLoadStatefulSet(@TempDir tempDir: Path) { + copyTestResourceFile("test-statefulset.yaml", tempDir) + val resourceSet = FileSystemResourceSet() - resourceSet.path = testResourcePath + resourceSet.path = tempDir.toString() resourceSet.files = listOf("test-statefulset.yaml") - assertEquals(1,resourceSet.getResourceSet(server.client).size) - assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is StatefulSet) + assertEquals(1, resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toList().first().second is StatefulSet) } @Test - fun testLoadConfigMap() { + fun testLoadConfigMap(@TempDir tempDir: Path) { + copyTestResourceFile("test-configmap.yaml", tempDir) + val resourceSet = FileSystemResourceSet() - resourceSet.path = testResourcePath + resourceSet.path = tempDir.toString() resourceSet.files = listOf("test-configmap.yaml") - assertEquals(1,resourceSet.getResourceSet(server.client).size) - assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is ConfigMap) + assertEquals(1, resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toList().first().second is ConfigMap) } @Test - fun testLoadServiceMonitor() { + fun testLoadServiceMonitor(@TempDir tempDir: Path) { + val serviceMonitorContext = ResourceDefinitionContext.Builder() + .withGroup("monitoring.coreos.com") + .withKind("ServiceMonitor") + .withPlural("servicemonitors") + .withNamespaced(true) + .withVersion("v1") + .build() + server.registerResource(serviceMonitorContext) + + copyTestResourceFile("test-service-monitor.yaml", tempDir) + val resourceSet = FileSystemResourceSet() - resourceSet.path = testResourcePath + resourceSet.path = tempDir.toString() resourceSet.files = listOf("test-service-monitor.yaml") - assertEquals(1,resourceSet.getResourceSet(server.client).size) - assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is CustomResourceWrapper) + assertEquals(1, resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toList().first().second is GenericKubernetesResource) } @Test - fun testLoadBenchmark() { + fun testLoadBenchmark(@TempDir tempDir: Path) { + copyTestResourceFile("test-benchmark.yaml", tempDir) + val resourceSet = FileSystemResourceSet() - resourceSet.path = testResourcePath + resourceSet.path = tempDir.toString() resourceSet.files = listOf("test-benchmark.yaml") - assertEquals(1,resourceSet.getResourceSet(server.client).size) - assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is CustomResourceWrapper) + assertEquals(1, resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toList().first().second is BenchmarkCRD) } @Test - fun testLoadExecution() { + fun testLoadExecution(@TempDir tempDir: Path) { + copyTestResourceFile("test-execution.yaml", tempDir) + val resourceSet = FileSystemResourceSet() - resourceSet.path = testResourcePath + resourceSet.path = tempDir.toString() resourceSet.files = listOf("test-execution.yaml") - assertEquals(1,resourceSet.getResourceSet(server.client).size) - assertTrue(resourceSet.getResourceSet(server.client).toMutableSet().first().second is CustomResourceWrapper) + assertEquals(1, resourceSet.getResourceSet(server.client).size) + assertTrue(resourceSet.getResourceSet(server.client).toList().first().second is ExecutionCRD) } @Test - fun testFilesNotSet() { + fun testFilesNotSet(@TempDir tempDir: Path) { + copyTestResourceFile("test-deployment.yaml", tempDir) + copyTestResourceFile("test-service.yaml", tempDir) + val resourceSet = FileSystemResourceSet() - resourceSet.path = testResourcePath - assertEquals(9,resourceSet.getResourceSet(server.client).size) + resourceSet.path = tempDir.toString() + assertEquals(2, resourceSet.getResourceSet(server.client).size) } @Test - fun testWrongPath() { + fun testWrongPath(@TempDir tempDir: Path) { val resourceSet = FileSystemResourceSet() - resourceSet.path = "/abc/not-exist" + resourceSet.path = "/not/existing/path" assertThrows<DeploymentFailedException> { resourceSet.getResourceSet(server.client) } diff --git a/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt b/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt index 138f79eadc6bdee17e62cc7a961eb7de539fa3df..ba65356b65b6ac23ca56c268bb003815917cf162 100644 --- a/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt +++ b/theodolite/src/test/kotlin/theodolite/execution/operator/StateHandlerTest.kt @@ -11,14 +11,12 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test import theodolite.k8s.K8sManager -import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile +import theodolite.model.crd.ExecutionCRD import theodolite.model.crd.ExecutionState -import java.time.Duration @QuarkusTest @WithKubernetesTestServer class StateHandlerTest { - private val testResourcePath = "./src/test/resources/k8s-resource-files/" @KubernetesTestServer private lateinit var server: KubernetesServer @@ -26,8 +24,8 @@ class StateHandlerTest { @BeforeEach fun setUp() { server.before() - val executionResource = K8sResourceLoaderFromFile(server.client) - .loadK8sResource("Execution", testResourcePath + "test-execution.yaml") + val executionStream = javaClass.getResourceAsStream("/k8s-resource-files/test-execution.yaml") + val executionResource = server.client.resources(ExecutionCRD::class.java).load(executionStream).get() K8sManager(server.client).deploy(executionResource) } diff --git a/theodolite/src/test/kotlin/theodolite/k8s/K8sManagerTest.kt b/theodolite/src/test/kotlin/theodolite/k8s/K8sManagerTest.kt index ffc3f2f2b8083ab8b8170fa77c19de3a6ef387e7..ee80d55caf995642f6fff04cfeeb66bc08ab93d3 100644 --- a/theodolite/src/test/kotlin/theodolite/k8s/K8sManagerTest.kt +++ b/theodolite/src/test/kotlin/theodolite/k8s/K8sManagerTest.kt @@ -1,32 +1,31 @@ package theodolite.k8s -import com.fasterxml.jackson.annotation.JsonIgnoreProperties import io.fabric8.kubernetes.api.model.* import io.fabric8.kubernetes.api.model.apps.Deployment import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder import io.fabric8.kubernetes.api.model.apps.StatefulSet import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder +import io.fabric8.kubernetes.client.dsl.base.ResourceDefinitionContext import io.fabric8.kubernetes.client.server.mock.KubernetesServer import io.quarkus.test.junit.QuarkusTest -import org.json.JSONObject -import org.junit.jupiter.api.AfterEach +import io.quarkus.test.kubernetes.client.KubernetesTestServer +import io.quarkus.test.kubernetes.client.WithKubernetesTestServer import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test -import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile +import registerResource + @QuarkusTest -@JsonIgnoreProperties(ignoreUnknown = true) -class K8sManagerTest { - @JsonIgnoreProperties(ignoreUnknown = true) - private final val server = KubernetesServer(false, true) - private final val testResourcePath = "./src/test/resources/k8s-resource-files/" +@WithKubernetesTestServer +internal class K8sManagerTest { + + @KubernetesTestServer + private lateinit var server: KubernetesServer private final val resourceName = "test-resource" private final val metadata: ObjectMeta = ObjectMetaBuilder().withName(resourceName).build() - val defaultDeployment: Deployment = DeploymentBuilder() .withMetadata(metadata) .withNewSpec() @@ -53,18 +52,6 @@ class K8sManagerTest { .withMetadata(metadata) .build() - @BeforeEach - fun setUp() { - server.before() - - } - - @AfterEach - fun tearDown() { - server.after() - - } - @Test @DisplayName("Test handling of Deployments") fun handleDeploymentTest() { @@ -121,32 +108,29 @@ class K8sManagerTest { @Test @DisplayName("Test handling of custom resources") fun handleCustomResourcesTest() { - val manager = K8sManager(server.client) - val servicemonitor = K8sResourceLoaderFromFile(server.client) - .loadK8sResource("ServiceMonitor", testResourcePath + "test-service-monitor.yaml") + val serviceMonitorContext = ResourceDefinitionContext.Builder() + .withGroup("monitoring.coreos.com") + .withKind("ServiceMonitor") + .withPlural("servicemonitors") + .withNamespaced(true) + .withVersion("v1") + .build() + server.registerResource(serviceMonitorContext) - val serviceMonitorContext = K8sContextFactory().create( - api = "v1", - scope = "Namespaced", - group = "monitoring.coreos.com", - plural = "servicemonitors" - ) - manager.deploy(servicemonitor) + val manager = K8sManager(server.client) - var serviceMonitors = JSONObject(server.client.customResource(serviceMonitorContext).list()) - .getJSONArray("items") + val serviceMonitorStream = javaClass.getResourceAsStream("/k8s-resource-files/test-service-monitor.yaml") + val serviceMonitor = server.client.load(serviceMonitorStream).get()[0] - assertEquals(1, serviceMonitors.length()) - assertEquals( - "test-service-monitor", - serviceMonitors.getJSONObject(0).getJSONObject("metadata").getString("name") - ) + manager.deploy(serviceMonitor) - manager.remove(servicemonitor) + val serviceMonitorsDeployed = server.client.genericKubernetesResources(serviceMonitorContext).list() + assertEquals(1, serviceMonitorsDeployed.items.size) + assertEquals("test-service-monitor", serviceMonitorsDeployed.items[0].metadata.name) - serviceMonitors = JSONObject(server.client.customResource(serviceMonitorContext).list()) - .getJSONArray("items") + manager.remove(serviceMonitor) - assertEquals(0, serviceMonitors.length()) + val serviceMonitorsDeleted = server.client.genericKubernetesResources(serviceMonitorContext).list() + assertEquals(0, serviceMonitorsDeleted.items.size) } } \ No newline at end of file diff --git a/theodolite/src/test/kotlin/theodolite/k8s/K8sResourceLoaderTest.kt b/theodolite/src/test/kotlin/theodolite/k8s/K8sResourceLoaderTest.kt deleted file mode 100644 index 4a41dac8b27b9d4ddcfb9915f759b14ea4eaba4a..0000000000000000000000000000000000000000 --- a/theodolite/src/test/kotlin/theodolite/k8s/K8sResourceLoaderTest.kt +++ /dev/null @@ -1,111 +0,0 @@ -package theodolite.k8s - -import io.fabric8.kubernetes.api.model.ConfigMap -import io.fabric8.kubernetes.api.model.Service -import io.fabric8.kubernetes.api.model.apps.Deployment -import io.fabric8.kubernetes.api.model.apps.StatefulSet -import io.fabric8.kubernetes.client.server.mock.KubernetesServer -import io.quarkus.test.junit.QuarkusTest -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.DisplayName -import org.junit.jupiter.api.Test -import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile - -@QuarkusTest -class K8sResourceLoaderTest { - private final val server = KubernetesServer(false, true) - private final val testResourcePath = "./src/test/resources/k8s-resource-files/" - - @BeforeEach - fun setUp() { - server.before() - } - - @AfterEach - fun tearDown() { - server.after() - } - - @Test - @DisplayName("Test loading of Deployments") - fun loadDeploymentTest() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("Deployment", testResourcePath + "test-deployment.yaml") - - assertTrue(resource is Deployment) - assertTrue(resource.toString().contains("name=test-deployment")) - } - - @Test - @DisplayName("Test loading of StatefulSet") - fun loadStatefulSetTest() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("StatefulSet", testResourcePath + "test-statefulset.yaml") - - assertTrue(resource is StatefulSet) - assertTrue(resource.toString().contains("name=test-statefulset")) - } - - @Test - @DisplayName("Test loading of Service") - fun loadServiceTest() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("Service", testResourcePath + "test-service.yaml") - - assertTrue(resource is Service) - assertTrue(resource.toString().contains("name=test-service")) - } - - @Test - @DisplayName("Test loading of ConfigMap") - fun loadConfigMapTest() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("ConfigMap", testResourcePath + "test-configmap.yaml") - - assertTrue(resource is ConfigMap) - assertTrue(resource.toString().contains("name=test-configmap")) - } - - @Test - @DisplayName("Test loading of ServiceMonitors") - fun loadServiceMonitorTest() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("ServiceMonitor", testResourcePath + "test-service-monitor.yaml") - - assertTrue(resource is CustomResourceWrapper) - if (resource is CustomResourceWrapper) { - assertEquals("test-service-monitor", resource.getName()) - - } - } - - @Test - @DisplayName("Test loading of Executions") - fun loadExecutionTest() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("Execution", testResourcePath + "test-execution.yaml") - - assertTrue(resource is CustomResourceWrapper) - if (resource is CustomResourceWrapper) { - assertEquals("example-execution", resource.getName()) - - } - } - - @Test - @DisplayName("Test loading of Benchmarks") - fun loadBenchmarkTest() { - val loader = K8sResourceLoaderFromFile(server.client) - val resource = loader.loadK8sResource("Benchmark", testResourcePath + "test-benchmark.yaml") - - assertTrue(resource is CustomResourceWrapper) - if (resource is CustomResourceWrapper) { - assertEquals("example-benchmark", resource.getName()) - - } - } - -} \ No newline at end of file diff --git a/theodolite/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt b/theodolite/src/test/kotlin/theodolite/patcher/ResourceLimitPatcherTest.kt similarity index 69% rename from theodolite/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt rename to theodolite/src/test/kotlin/theodolite/patcher/ResourceLimitPatcherTest.kt index b7fc2d9f1b2d5110f974b3805584baa3903d5eb1..2769f2fef607a03d820b0821969db98894944cb3 100644 --- a/theodolite/src/test/kotlin/theodolite/ResourceLimitPatcherTest.kt +++ b/theodolite/src/test/kotlin/theodolite/patcher/ResourceLimitPatcherTest.kt @@ -1,11 +1,12 @@ -package theodolite +package theodolite.patcher -import io.fabric8.kubernetes.api.model.apps.Deployment -import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.server.mock.KubernetesServer import io.quarkus.test.junit.QuarkusTest +import io.quarkus.test.kubernetes.client.KubernetesTestServer +import io.quarkus.test.kubernetes.client.WithKubernetesTestServer import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test -import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile import theodolite.patcher.PatcherFactory import theodolite.util.PatcherDefinition @@ -20,40 +21,44 @@ import theodolite.util.PatcherDefinition * Case 4: In the given YAML declaration neither `Resource Request` nor `Request Limit` is defined */ @QuarkusTest +@WithKubernetesTestServer +@Disabled class ResourceLimitPatcherTest { - val testPath = "./src/test/resources/" - val loader = K8sResourceLoaderFromFile(DefaultKubernetesClient().inNamespace("")) + val patcherFactory = PatcherFactory() + @KubernetesTestServer + private lateinit var server: KubernetesServer + fun applyTest(fileName: String) { val cpuValue = "50m" val memValue = "3Gi" - val k8sResource = loader.loadK8sResource("Deployment", testPath + fileName) as Deployment + val k8sResource = server.client.apps().deployments().load(javaClass.getResourceAsStream(fileName)).get() val defCPU = PatcherDefinition() - defCPU.resource = "cpu-memory-deployment.yaml" + defCPU.resource = "/cpu-memory-deployment.yaml" defCPU.type = "ResourceLimitPatcher" - defCPU.properties = mutableMapOf( + defCPU.properties = mapOf( "limitedResource" to "cpu", "container" to "application" ) val defMEM = PatcherDefinition() - defMEM.resource = "cpu-memory-deployment.yaml" + defMEM.resource = "/cpu-memory-deployment.yaml" defMEM.type = "ResourceLimitPatcher" - defMEM.properties = mutableMapOf( + defMEM.properties = mapOf( "limitedResource" to "memory", "container" to "uc-application" ) patcherFactory.createPatcher( patcherDefinition = defCPU, - k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)) + k8sResources = listOf(Pair("/cpu-memory-deployment.yaml", k8sResource)) ).patch(value = cpuValue) patcherFactory.createPatcher( patcherDefinition = defMEM, - k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)) + k8sResources = listOf(Pair("/cpu-memory-deployment.yaml", k8sResource)) ).patch(value = memValue) k8sResource.spec.template.spec.containers.filter { it.name == defCPU.properties["container"]!! } @@ -66,24 +71,24 @@ class ResourceLimitPatcherTest { @Test fun testWithExistingCpuAndMemoryDeclarations() { // Case 1: In the given YAML declaration memory and cpu are defined - applyTest("cpu-memory-deployment.yaml") + applyTest("/cpu-memory-deployment.yaml") } @Test fun testOnlyWithExistingCpuDeclarations() { // Case 2: In the given YAML declaration only cpu is defined - applyTest("cpu-deployment.yaml") + applyTest("/cpu-deployment.yaml") } @Test fun testOnlyWithExistingMemoryDeclarations() { // Case 3: In the given YAML declaration only memory is defined - applyTest("memory-deployment.yaml") + applyTest("/memory-deployment.yaml") } @Test fun testWithoutResourceDeclarations() { // Case 4: In the given YAML declaration neither `Resource Request` nor `Request Limit` is defined - applyTest("no-resources-deployment.yaml") + applyTest("/no-resources-deployment.yaml") } } diff --git a/theodolite/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt b/theodolite/src/test/kotlin/theodolite/patcher/ResourceRequestPatcherTest.kt similarity index 69% rename from theodolite/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt rename to theodolite/src/test/kotlin/theodolite/patcher/ResourceRequestPatcherTest.kt index 8794d4dc2d67b8af78f4fa409c727f882922d0b8..dba91eb65d4474d38f64d7fdd7f7ab981f8eb30f 100644 --- a/theodolite/src/test/kotlin/theodolite/ResourceRequestPatcherTest.kt +++ b/theodolite/src/test/kotlin/theodolite/patcher/ResourceRequestPatcherTest.kt @@ -1,12 +1,11 @@ -package theodolite +package theodolite.patcher -import io.fabric8.kubernetes.api.model.apps.Deployment -import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.server.mock.KubernetesServer import io.quarkus.test.junit.QuarkusTest +import io.quarkus.test.kubernetes.client.KubernetesTestServer +import io.quarkus.test.kubernetes.client.WithKubernetesTestServer import io.smallrye.common.constraint.Assert.assertTrue import org.junit.jupiter.api.Test -import theodolite.k8s.resourceLoader.K8sResourceLoaderFromFile -import theodolite.patcher.PatcherFactory import theodolite.util.PatcherDefinition /** @@ -20,39 +19,42 @@ import theodolite.util.PatcherDefinition * Case 4: In the given YAML declaration neither `Resource Request` nor `Request Limit` is defined */ @QuarkusTest +@WithKubernetesTestServer class ResourceRequestPatcherTest { - val testPath = "./src/test/resources/" - val loader = K8sResourceLoaderFromFile(DefaultKubernetesClient().inNamespace("")) + + @KubernetesTestServer + private lateinit var server: KubernetesServer + val patcherFactory = PatcherFactory() fun applyTest(fileName: String) { val cpuValue = "50m" val memValue = "3Gi" - val k8sResource = loader.loadK8sResource("Deployment", testPath + fileName) as Deployment + val k8sResource = server.client.apps().deployments().load(javaClass.getResourceAsStream(fileName)).get() val defCPU = PatcherDefinition() - defCPU.resource = "cpu-memory-deployment.yaml" + defCPU.resource = "/cpu-memory-deployment.yaml" defCPU.type = "ResourceRequestPatcher" - defCPU.properties = mutableMapOf( + defCPU.properties = mapOf( "requestedResource" to "cpu", "container" to "application" ) val defMEM = PatcherDefinition() - defMEM.resource = "cpu-memory-deployment.yaml" + defMEM.resource = "/cpu-memory-deployment.yaml" defMEM.type = "ResourceRequestPatcher" - defMEM.properties = mutableMapOf( + defMEM.properties = mapOf( "requestedResource" to "memory", "container" to "application" ) patcherFactory.createPatcher( patcherDefinition = defCPU, - k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)) + k8sResources = listOf(Pair("/cpu-memory-deployment.yaml", k8sResource)) ).patch(value = cpuValue) patcherFactory.createPatcher( patcherDefinition = defMEM, - k8sResources = listOf(Pair("cpu-memory-deployment.yaml", k8sResource)) + k8sResources = listOf(Pair("/cpu-memory-deployment.yaml", k8sResource)) ).patch(value = memValue) k8sResource.spec.template.spec.containers.filter { it.name == defCPU.properties["container"]!! } @@ -65,24 +67,24 @@ class ResourceRequestPatcherTest { @Test fun testWithExistingCpuAndMemoryDeclarations() { // Case 1: In the given YAML declaration memory and cpu are defined - applyTest("cpu-memory-deployment.yaml") + applyTest("/cpu-memory-deployment.yaml") } @Test fun testOnlyWithExistingCpuDeclarations() { // Case 2: In the given YAML declaration only cpu is defined - applyTest("cpu-deployment.yaml") + applyTest("/cpu-deployment.yaml") } @Test fun testOnlyWithExistingMemoryDeclarations() { // Case 3: In the given YAML declaration only memory is defined - applyTest("memory-deployment.yaml") + applyTest("/memory-deployment.yaml") } @Test fun testWithoutResourceDeclarations() { // Case 4: In the given YAML declaration neither `Resource Request` nor `Request Limit` is defined - applyTest("no-resources-deployment.yaml") + applyTest("/no-resources-deployment.yaml") } } diff --git a/theodolite/src/test/kotlin/theodolite/util/IOHandlerTest.kt b/theodolite/src/test/kotlin/theodolite/util/IOHandlerTest.kt index f84536bfc029a829c1798293938386965eedcf47..3b31f389bdeb35e6016a56a98abb1e13bf83ae18 100644 --- a/theodolite/src/test/kotlin/theodolite/util/IOHandlerTest.kt +++ b/theodolite/src/test/kotlin/theodolite/util/IOHandlerTest.kt @@ -12,7 +12,6 @@ import org.junit.rules.TemporaryFolder import org.junitpioneer.jupiter.ClearEnvironmentVariable import org.junitpioneer.jupiter.SetEnvironmentVariable - const val FOLDER_URL = "Test-Folder" @QuarkusTest @@ -56,11 +55,12 @@ internal class IOHandlerTest { columns = columns ) - var expected = "Fruit,Color\n" - testContent.forEach { expected += it[0] + "," + it[1] + "\n" } + val expected = (listOf(listOf("Fruit", "Color")) + testContent) + .map { "${it[0]},${it[1]}" } + .reduce { left, right -> left + System.lineSeparator() + right } assertEquals( - expected.trim(), + expected, IOHandler().readFileAsString("${folder.absolutePath}/test-file.csv") ) } diff --git a/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml b/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml index 1ba204bb2821f9b734706d322322b28220ef19d5..ea9ee8471d3da1dc6011348bd978696bd0fa6f36 100644 --- a/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml +++ b/theodolite/src/test/resources/k8s-resource-files/test-benchmark.yaml @@ -3,14 +3,19 @@ kind: benchmark metadata: name: example-benchmark spec: - appResource: - - "uc1-kstreams-deployment.yaml" - - "aggregation-service.yaml" - - "jmx-configmap.yaml" - - "uc1-service-monitor.yaml" - loadGenResource: - - "uc1-load-generator-deployment.yaml" - - "uc1-load-generator-service.yaml" + sut: + resources: + - configMap: + name: "example-configmap" + files: + - "uc1-kstreams-deployment.yaml" + loadGenerator: + resources: + - configMap: + name: "example-configmap" + files: + - uc1-load-generator-service.yaml + - uc1-load-generator-deployment.yaml resourceTypes: - typeName: "Instances" patchers: @@ -35,4 +40,4 @@ spec: numPartitions: 40 replicationFactor: 1 - name: "theodolite-.*" - removeOnly: True \ No newline at end of file + removeOnly: True