diff --git a/helm/templates/kafka/kafka-cluster.yaml b/helm/templates/kafka/kafka-cluster.yaml index f1a58077a78865c624706531b58c3150feeb83ae..ee2bd0f1c529f98a9d838a704c6f277e9106f439 100644 --- a/helm/templates/kafka/kafka-cluster.yaml +++ b/helm/templates/kafka/kafka-cluster.yaml @@ -30,14 +30,9 @@ spec: configMapKeyRef: name: {{ template "theodolite.fullname" . }}-kafka-metrics key: kafka-metrics-config.yml - {{- with .Values.strimzi.kafka.nodeSelectorTerms}} + {{- with .Values.strimzi.kafka.template}} template: - pod: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - {{- toYaml . | nindent 16 }} + {{- toYaml . | nindent 6 }} {{- end}} {{- with .Values.strimzi.kafka.resources}} resources: @@ -52,14 +47,9 @@ spec: {{- end }} storage: type: ephemeral - {{- with .Values.strimzi.zookeeper.nodeSelectorTerms}} + {{- with .Values.strimzi.kafka.template}} template: - pod: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - {{- toYaml . | nindent 16 }} + {{- toYaml . | nindent 6 }} {{- end}} kafkaExporter: {} diff --git a/helm/templates/kafka/schema-registry-deployment.yaml b/helm/templates/kafka/schema-registry-deployment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e59acdb7eba7856c7f28f625e4fa2de42a9025b3 --- /dev/null +++ b/helm/templates/kafka/schema-registry-deployment.yaml @@ -0,0 +1,59 @@ +{{- if .Values.strimzi.schemaRegistry.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "theodolite.fullname" . }}-kafka-schema-registry + labels: + app: schema-registry +spec: + replicas: {{ .Values.strimzi.schemaRegistry.replicaCount }} + selector: + matchLabels: + app: schema-registry + template: + metadata: + labels: + app: schema-registry + spec: + securityContext: + {{- if .Values.strimzi.schemaRegistry.securityContext }} +{{ toYaml .Values.strimzi.schemaRegistry.securityContext | indent 8 }} + {{- end}} + containers: + - name: {{ template "theodolite.fullname" . }}-kafka-schema-registry-server + image: "{{ .Values.strimzi.schemaRegistry.image }}:{{ .Values.strimzi.schemaRegistry.imageTag }}" + imagePullPolicy: "{{ .Values.strimzi.schemaRegistry.imagePullPolicy }}" + ports: + - name: schema-registry + containerPort: {{ .Values.strimzi.schemaRegistry.servicePort }} + protocol: TCP + resources: +{{ toYaml .Values.strimzi.schemaRegistry.resources | indent 12 }} + env: + - name: SCHEMA_REGISTRY_HOST_NAME + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: SCHEMA_REGISTRY_LISTENERS + value: http://0.0.0.0:{{ .Values.strimzi.schemaRegistry.servicePort }} + - name: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS + value: {{ template "theodolite.fullname" . }}-kafka-kafka-bootstrap:9092 + - name: SCHEMA_REGISTRY_KAFKASTORE_GROUP_ID + value: {{ template "theodolite.fullname" . }} + - name: SCHEMA_REGISTRY_MASTER_ELIGIBILITY + value: "true" + - name: SCHEMA_REGISTRY_HEAP_OPTS + value: "{{ .Values.strimzi.schemaRegistry.heapOptions }}" + {{- if .Values.strimzi.schemaRegistry.nodeSelector }} + nodeSelector: +{{ toYaml .Values.strimzi.schemaRegistry.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.strimzi.schemaRegistry.tolerations }} + tolerations: +{{ toYaml .Values.strimzi.schemaRegistry.tolerations | indent 8 }} + {{- end }} + {{- if .Values.strimzi.schemaRegistry.affinity }} + affinity: +{{ toYaml .Values.strimzi.schemaRegistry.affinity | indent 8 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/templates/kafka/schema-registry-service.yaml b/helm/templates/kafka/schema-registry-service.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccee27f47dc12066fa175ca885850f133a2d8441 --- /dev/null +++ b/helm/templates/kafka/schema-registry-service.yaml @@ -0,0 +1,14 @@ +{{- if .Values.strimzi.schemaRegistry.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "theodolite.fullname" . }}-kafka-schema-registry + labels: + app: schema-registry +spec: + ports: + - name: schema-registry + port: {{ .Values.strimzi.schemaRegistry.servicePort }} + selector: + app: schema-registry +{{- end }} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index eb4cfc4b29fee97ab2be88a3ce2ded766a122b20..995408344d69352136c0569056a24e0f8dec4c10 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -122,7 +122,7 @@ cp-helm-charts: ## Schema Registry ## ------------------------------------------------------ cp-schema-registry: - enabled: true + enabled: false nodeSelector: {} image: confluentinc/cp-schema-registry imageTag: 5.4.0 @@ -194,6 +194,26 @@ strimzi: topicOperator: enabled: true + schemaRegistry: + enabled: true + replicaCount: 1 + ## ref: https://hub.docker.com/r/confluentinc/cp-schema-registry/ + image: confluentinc/cp-schema-registry + #imageTag: 6.1.0 + imageTag: 5.4.0 + imagePullPolicy: IfNotPresent + servicePort: 8081 + heapOptions: "-Xms512M -Xmx512M" + resources: {} + nodeSelector: {} + tolerations: [] + affinity: {} + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + runAsNonRoot: true + ### # Prometheus Monitoring Stack (Prometheus Operator) diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml index c1fb35cbfb67efa5b4e41787c342374e835a1f3e..d274cba8a326ddaac7670e7797b899d6022b12c9 100644 --- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml @@ -27,7 +27,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml index 7d70908589cfa2124af64cf3c671ed62c00b1be6..bcf3a04f34dd039a3d75c86df095d09b9523f615 100644 --- a/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml @@ -23,7 +23,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml index 607767382b4b2fce406fe3820c0f8c6ed1cef777..d7f453e2ab9c15bca83162520ebe7e90ce7be3ea 100644 --- a/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml @@ -28,7 +28,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" # - name: JAVA_OPTS # value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - name: COMMIT_INTERVAL_MS # Set as default for the applications diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml index 8e0114f17ae085057382eee597ad275c6fc0329d..74c5002dbdf988071ee43946b55b56d7d439a09e 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml @@ -22,7 +22,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml index 9710d44cbb1ffb75e4d1ebafe1ffe60042587adc..be23c1d6abf2ccdc39481997bff12f11fd8fd580 100644 --- a/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml @@ -22,7 +22,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml index cab5a625c323628ec6d6a7152e53d3ff8393a8ba..6432679b05264150ccccc84490c2badb466d9beb 100644 --- a/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml @@ -21,7 +21,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" - name: KUBERNETES_NAMESPACE diff --git a/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml index b4d2bfe738fd9c96d0219a825d5c4ef2cf5385f3..7698dc50306901d8fd7e2f4ab79f2f362b909d54 100644 --- a/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml @@ -23,7 +23,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: JAVA_OPTS value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - name: COMMIT_INTERVAL_MS # Set as default for the applications diff --git a/theodolite-benchmarks/definitions/uc1-load-generator/resources/uc1-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc1-load-generator/resources/uc1-load-generator-deployment.yaml index 65048a97d5de3d831f782db329e295a5e5ceb727..cd408e786204d3dcd466af2f998a242832c1727d 100644 --- a/theodolite-benchmarks/definitions/uc1-load-generator/resources/uc1-load-generator-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc1-load-generator/resources/uc1-load-generator-deployment.yaml @@ -29,4 +29,4 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-deployment.yaml index 79d69aa0cfe77d80e70ff9f1d146790bce0c4577..667559010b0dc9f642f9cb0e8c9d257d70a2dbaa 100644 --- a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-deployment.yaml @@ -27,7 +27,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml index 86e17e96e83f991bd0b6c7bd4576d100c9222cb1..f4f7a9c54a55c42dffe838925a8a97e73ef74759 100644 --- a/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml @@ -23,7 +23,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml index bcb7d82ed463a194fb0851287487a6d52cfd0198..4266b47a15f2b43e7d746a581c1cd5b1f7321096 100644 --- a/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml @@ -28,7 +28,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" resources: diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml index f2fc1ee6a3fa01d367c52505c49bda1e58ddc134..d9d7182837dcd29dec80e74d631270b678dc9a7f 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml @@ -22,7 +22,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml index b9a917ee825637988cf6902186d99421aac3d671..add5ee6c770f8a4fa1cd8685009ae0fba85ac459 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml @@ -22,7 +22,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml index 0f5000902ddf9c12f67643cc35ffc3c882970a72..fddfa7512a41c75321e49272a41824383b4906e8 100644 --- a/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml @@ -21,7 +21,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" - name: DOWNSAMPLE_INTERVAL diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-kstreams-deployment.yaml index 86932cafb26248736fbe060ba7f23ee5dded412d..47711343508c712e8f46d38feaef6b6057fbdc80 100644 --- a/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-kstreams-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-kstreams-deployment.yaml @@ -23,7 +23,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: JAVA_OPTS value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - name: COMMIT_INTERVAL_MS # Set as default for the applications diff --git a/theodolite-benchmarks/definitions/uc2-load-generator/resources/uc2-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc2-load-generator/resources/uc2-load-generator-deployment.yaml index d758c66f88fa93c98258febf6c5e6a35f7171820..b3eb43070493a81d3ff717d5d16541a9c03ccc3a 100644 --- a/theodolite-benchmarks/definitions/uc2-load-generator/resources/uc2-load-generator-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc2-load-generator/resources/uc2-load-generator-deployment.yaml @@ -29,4 +29,4 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-deployment.yaml index c44f32f0846de4b32afeba2344abcd6f805c38b2..1e70c714d0feef09e034566e9e1745b3b310753b 100644 --- a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-deployment.yaml @@ -27,7 +27,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml index a0a840fad96fa21212bb25c78a597ea5cb41558c..e71177f1c7d3595b4dc48a82aebfe839bc2786eb 100644 --- a/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-beam-flink/resources/taskmanager-deployment.yaml @@ -23,7 +23,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml index aff9da6a0f7ee490a94a6f9954c2b3cc8a2b5ed5..bb1917a156d6a04277a3ca6e3bccd580d760de21 100644 --- a/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-beam-samza/resources/uc3-beam-samza-deployment.yaml @@ -28,7 +28,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" resources: diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml index f1c56b3a51ec884dca25a31ffafea195919a02e2..90e13d000d48be220d40027aaf648d7e0472bbff 100644 --- a/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-flink/resources/jobmanager-deployment.yaml @@ -22,7 +22,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml index cc1efa23c32220c7c664d8aaa4669f3af6492d15..7092166f295a8bb85f929775170b67c41cf17612 100644 --- a/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-flink/resources/taskmanager-deployment.yaml @@ -22,7 +22,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml index 317d30328ec9ba3c0c30bf87733b3801e73d2477..f1c6cb366417e3ba8daf4cdcd7454b890d3c45e0 100644 --- a/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-hazelcastjet/resources/uc3-hazelcastjet-deployment.yaml @@ -21,7 +21,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" - name: WINDOW_SIZE_IN_SECONDS diff --git a/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-kstreams-deployment.yaml index 0b58c15fd123c9d06e441084063eae40d20cc48a..1a98c1a5edf2b4b0dc3b8fbb155c56c347ce22fa 100644 --- a/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-kstreams-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-kstreams/resources/uc3-kstreams-deployment.yaml @@ -23,7 +23,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: JAVA_OPTS value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - name: COMMIT_INTERVAL_MS # Set as default for the applications diff --git a/theodolite-benchmarks/definitions/uc3-load-generator/resources/uc3-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc3-load-generator/resources/uc3-load-generator-deployment.yaml index 928e0a91b210e786b3d6b156e964a7fb9a7cc184..a8e13a543075cd6a48a73354f9beac7e59bfd27a 100644 --- a/theodolite-benchmarks/definitions/uc3-load-generator/resources/uc3-load-generator-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc3-load-generator/resources/uc3-load-generator-deployment.yaml @@ -29,4 +29,4 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-deployment.yaml index fb54893cc8dcea45fd6c657d25eb97b40399a29e..2cd558313e07ed35945efcb2a16528911bdf3121 100644 --- a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/jobmanager-deployment.yaml @@ -27,7 +27,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml index 5eabd7d52875b8ea556a4b47fde18cf859a279b7..75718b28564a2ffca7f684eccaab26b9caff6a5f 100644 --- a/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-beam-flink/resources/taskmanager-deployment.yaml @@ -23,7 +23,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml index c0602d86207d2e63d3eb549b068b0afe2692df21..fccf67c82bbda59a1f31fa0617cd158a99a09f46 100644 --- a/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-beam-samza/resources/uc4-beam-samza-deployment.yaml @@ -28,7 +28,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" resources: diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml index d7037e2c579d82485bb31c53f132d7938f424b38..fe8f232170406d06f7339455e1488c716d0326db 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/resources/jobmanager-deployment.yaml @@ -22,7 +22,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml index cc1efa23c32220c7c664d8aaa4669f3af6492d15..7092166f295a8bb85f929775170b67c41cf17612 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/resources/taskmanager-deployment.yaml @@ -22,7 +22,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS value: "100" - name: CHECKPOINTING diff --git a/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml b/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml index f0736e585400d26481272d5b3d75cd216d55527d..ab872b889877d7787e30a2142e1c2c5b564b5235 100644 --- a/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-hazelcastjet/resources/uc4-hazelcastjet-deployment.yaml @@ -21,7 +21,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: COMMIT_INTERVAL_MS # Set as default for the applications value: "100" - name: WINDOW_SIZE diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-kstreams-deployment.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-kstreams-deployment.yaml index 3c9a96e36ebf31397f91930426fc028be9d5a2c6..bc083914431fa027308205825730afd5a19cc250 100644 --- a/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-kstreams-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-kstreams/resources/uc4-kstreams-deployment.yaml @@ -23,7 +23,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: JAVA_OPTS value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - name: COMMIT_INTERVAL_MS # Set as default for the applications diff --git a/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml b/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml index 70674757ff97b4377902395868eac2e03070c31a..49757b35b1cd20a4699f3e6f546d3100e6e4c972 100644 --- a/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml +++ b/theodolite-benchmarks/definitions/uc4-load-generator/resources/uc4-load-generator-deployment.yaml @@ -29,7 +29,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: NUM_SENSORS value: "4" - name: NUM_NESTED_GROUPS diff --git a/theodolite/examples/operator/example-configmap.yaml b/theodolite/examples/operator/example-configmap.yaml index db511a94cc903869677f2e447d45baf0d983ac6c..585314e10fd1b3da4b154b7fb629e826a08147c6 100644 --- a/theodolite/examples/operator/example-configmap.yaml +++ b/theodolite/examples/operator/example-configmap.yaml @@ -26,7 +26,7 @@ data: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: JAVA_OPTS value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - name: COMMIT_INTERVAL_MS # Set as default for the applications @@ -67,7 +67,7 @@ data: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" uc1-load-generator-service.yaml: | apiVersion: v1 kind: Service diff --git a/theodolite/examples/resources/uc1-kstreams-deployment.yaml b/theodolite/examples/resources/uc1-kstreams-deployment.yaml index 1951b1177572dbd1276afcbc6770f91f9f5ff168..1c0b138bd48d71acb74c0b532848f06655c1d021 100644 --- a/theodolite/examples/resources/uc1-kstreams-deployment.yaml +++ b/theodolite/examples/resources/uc1-kstreams-deployment.yaml @@ -23,7 +23,7 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" - name: JAVA_OPTS value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555" - name: COMMIT_INTERVAL_MS # Set as default for the applications diff --git a/theodolite/examples/resources/uc1-load-generator-deployment.yaml b/theodolite/examples/resources/uc1-load-generator-deployment.yaml index 65048a97d5de3d831f782db329e295a5e5ceb727..cd408e786204d3dcd466af2f998a242832c1727d 100644 --- a/theodolite/examples/resources/uc1-load-generator-deployment.yaml +++ b/theodolite/examples/resources/uc1-load-generator-deployment.yaml @@ -29,4 +29,4 @@ spec: - name: KAFKA_BOOTSTRAP_SERVERS value: "theodolite-kafka-kafka-bootstrap:9092" - name: SCHEMA_REGISTRY_URL - value: "http://theodolite-cp-schema-registry:8081" + value: "http://theodolite-kafka-schema-registry:8081" diff --git a/theodolite/src/main/kotlin/rocks/theodolite/core/IOHandler.kt b/theodolite/src/main/kotlin/rocks/theodolite/core/IOHandler.kt index 4d2cab0da938b18950def8cfb5cc6f104e110125..e70779afb34c4e7cecd561a2c4d64b09c9933ad5 100644 --- a/theodolite/src/main/kotlin/rocks/theodolite/core/IOHandler.kt +++ b/theodolite/src/main/kotlin/rocks/theodolite/core/IOHandler.kt @@ -64,20 +64,46 @@ class IOHandler { * Write to CSV file * * @param fileURL the URL of the file - * @param data the data to write in the file, as list of list, each subList corresponds to a row in the CSV file - * @param columns columns of the CSV file + * @param data the data to write in the csv, as list of list, + * each sublist corresponds to a row in the CSV file + * @param columns name of the columns */ fun writeToCSVFile(fileURL: String, data: List<List<String>>, columns: List<String>) { val outputFile = File("$fileURL.csv") PrintWriter(outputFile).use { pw -> - pw.println(columns.joinToString(separator = ",")) - data.forEach { - pw.println(it.joinToString(separator = ",")) + + val writeColumns = addQuotationMarks(columns) + pw.println(writeColumns.joinToString(separator = ",")) + + data.forEach{ row -> + val writeRow = addQuotationMarks(row) + pw.println(writeRow.joinToString(separator = ",")) } } logger.info { "Wrote CSV file: $fileURL to ${outputFile.absolutePath}." } } + /** + * For a list of Strings: + * - adds additional quotation mark to existing one + * - adds quotation marks around entries that contain a comma + */ + private fun addQuotationMarks(stringList: List<String> ): List<String> { + val stringMutableList = stringList.toMutableList() + stringMutableList.forEachIndexed { index, entry -> + // add additional quotation marks to escape them in csv + if (entry.contains("\"")){ + stringMutableList[index] = stringMutableList[index].replace('"'+"", "\"" + '"') + } + + // add quotation marks around entries that contain a comma + if (entry.contains(",")){ + stringMutableList[index] = '"' + stringMutableList[index] + '"' + } + } + return stringMutableList + } + /** * Write to text file * diff --git a/theodolite/src/test/kotlin/rocks/theodolite/core/IOHandlerTest.kt b/theodolite/src/test/kotlin/rocks/theodolite/core/IOHandlerTest.kt index 65e84d7dd37eb5b68f77bc2d47d212db2f720a90..e54bf7851b195927f69666caa2fd3b33fb3673d7 100644 --- a/theodolite/src/test/kotlin/rocks/theodolite/core/IOHandlerTest.kt +++ b/theodolite/src/test/kotlin/rocks/theodolite/core/IOHandlerTest.kt @@ -11,6 +11,7 @@ import org.junit.jupiter.api.Test import org.junit.rules.TemporaryFolder import org.junitpioneer.jupiter.ClearEnvironmentVariable import org.junitpioneer.jupiter.SetEnvironmentVariable +import java.util.stream.Collectors const val FOLDER_URL = "Test-Folder" @@ -42,12 +43,13 @@ internal class IOHandlerTest { temporaryFolder.create() val folder = temporaryFolder.newFolder(FOLDER_URL) + val columns = listOf("Fruit", "Color") + val testContent = listOf( listOf("apples", "red"), listOf("bananas", "yellow"), listOf("avocado", "brown") ) - val columns = listOf("Fruit", "Color") IOHandler().writeToCSVFile( fileURL = "${folder.absolutePath}/test-file", @@ -65,6 +67,49 @@ internal class IOHandlerTest { ) } + /** + * Tests if values with commas and quotation marks are surrounded with additional quotation marks. + */ + @Test + fun testWriteToCSVFileWithComma() { + temporaryFolder.create() + val folder = temporaryFolder.newFolder(FOLDER_URL) + + val columns = listOf("Fruit, Fruit2", "Color") + val expectedColumns = listOf("\"Fruit, Fruit2\"", "Color") + + val testContent = listOf( + listOf("apples, "+ '"' + "paprika" + '"', "red"), + listOf("bananas, pineapple", "yellow"), + listOf("avocado, coconut", "brown") + ) + + val expectedContent = listOf( + listOf("\"apples, " + '"' + '"' + "paprika" + '"' + '"' + '"', "red"), + listOf("\"bananas, pineapple\"", "yellow"), + listOf("\"avocado, coconut\"", "brown") + ) + + IOHandler().writeToCSVFile( + fileURL = "${folder.absolutePath}/test-file", + data = testContent, + columns = columns + ) + + // construct string from the columns + var expected = expectedColumns.stream().collect(Collectors.joining(",")) + + // add values from the expectedContent to expected string + expectedContent.forEach{ + expected += "\n" + it.joinToString(separator = ",") + } + + assertEquals( + expected, + IOHandler().readFileAsString("${folder.absolutePath}/test-file.csv") + ) + } + @Test fun testWriteToJSONFile() { temporaryFolder.create()