Merge branch 'own-schema-registry' into 'master'

Introduce custom Schema Registry deployment

Closes #355

See merge request !287

helm/templates/kafka/schema-registry-deployment.yaml

+{{- if .Values.strimzi.schemaRegistry.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "theodolite.fullname" . }}-kafka-schema-registry
+  labels:
+    app: schema-registry
+spec:
+  replicas: {{ .Values.strimzi.schemaRegistry.replicaCount }}
+  selector:
+    matchLabels:
+      app: schema-registry
+  template:
+    metadata:
+      labels:
+        app: schema-registry
+    spec:
+      securityContext:
+      {{- if .Values.strimzi.schemaRegistry.securityContext }}
+{{ toYaml .Values.strimzi.schemaRegistry.securityContext | indent 8 }}
+      {{- end}}
+      containers:
+        - name: {{ template "theodolite.fullname" . }}-kafka-schema-registry-server
+          image: "{{ .Values.strimzi.schemaRegistry.image }}:{{ .Values.strimzi.schemaRegistry.imageTag }}"
+          imagePullPolicy: "{{ .Values.strimzi.schemaRegistry.imagePullPolicy }}"
+          ports:
+            - name: schema-registry
+              containerPort: {{ .Values.strimzi.schemaRegistry.servicePort }}
+              protocol: TCP
+          resources:
+{{ toYaml .Values.strimzi.schemaRegistry.resources | indent 12 }}
+          env:
+          - name: SCHEMA_REGISTRY_HOST_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: SCHEMA_REGISTRY_LISTENERS
+            value: http://0.0.0.0:{{ .Values.strimzi.schemaRegistry.servicePort }}
+          - name: SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS
+            value: {{ template "theodolite.fullname" . }}-kafka-kafka-bootstrap:9092
+          - name: SCHEMA_REGISTRY_KAFKASTORE_GROUP_ID
+            value: {{ template "theodolite.fullname" . }}
+          - name: SCHEMA_REGISTRY_MASTER_ELIGIBILITY
+            value: "true"
+          - name: SCHEMA_REGISTRY_HEAP_OPTS
+            value: "{{ .Values.strimzi.schemaRegistry.heapOptions }}"
+      {{- if .Values.strimzi.schemaRegistry.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.strimzi.schemaRegistry.nodeSelector | indent 8 }}
+      {{- end }}
+      {{- if .Values.strimzi.schemaRegistry.tolerations }}
+      tolerations:
+{{ toYaml .Values.strimzi.schemaRegistry.tolerations | indent 8 }}
+      {{- end }}
+      {{- if .Values.strimzi.schemaRegistry.affinity }}
+      affinity:
+{{ toYaml .Values.strimzi.schemaRegistry.affinity | indent 8 }}
+      {{- end }}
+{{- end }}
\ No newline at end of file

helm/templates/kafka/schema-registry-service.yaml

+{{- if .Values.strimzi.schemaRegistry.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "theodolite.fullname" . }}-kafka-schema-registry
+  labels:
+    app: schema-registry
+spec:
+  ports:
+    - name: schema-registry
+      port: {{ .Values.strimzi.schemaRegistry.servicePort }}
+  selector:
+    app: schema-registry
+{{- end }}
\ No newline at end of file

helm/values.yaml

@@ -122,7 +122,7 @@ cp-helm-charts:
   ## Schema Registry
   ## ------------------------------------------------------
   cp-schema-registry:
-    enabled: true
+    enabled: false
     nodeSelector: {}
     image: confluentinc/cp-schema-registry
     imageTag: 5.4.0
@@ -194,6 +194,26 @@ strimzi:
   topicOperator:
     enabled: true
 
+  schemaRegistry:
+    enabled: true
+    replicaCount: 1
+    ## ref: https://hub.docker.com/r/confluentinc/cp-schema-registry/
+    image: confluentinc/cp-schema-registry
+    #imageTag: 6.1.0
+    imageTag: 5.4.0
+    imagePullPolicy: IfNotPresent
+    servicePort: 8081
+    heapOptions: "-Xms512M -Xmx512M"
+    resources: {}
+    nodeSelector: {}
+    tolerations: []
+    affinity: {}
+    securityContext:
+      runAsUser: 10001
+      runAsGroup: 10001
+      fsGroup: 10001
+      runAsNonRoot: true
+
 
 ###
 # Prometheus Monitoring Stack (Prometheus Operator)

theodolite-benchmarks/definitions/uc1-beam-flink/resources/jobmanager-deployment.yaml

@@ -27,7 +27,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS
               value: "100"
             - name: CHECKPOINTING

theodolite-benchmarks/definitions/uc1-beam-flink/resources/taskmanager-deployment.yaml

@@ -23,7 +23,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS
               value: "100"
             - name: CHECKPOINTING

theodolite-benchmarks/definitions/uc1-beam-samza/resources/uc1-beam-samza-deployment.yaml

@@ -28,7 +28,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             # - name: JAVA_OPTS
             #   value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
             - name: COMMIT_INTERVAL_MS # Set as default for the applications

theodolite-benchmarks/definitions/uc1-flink/resources/jobmanager-deployment.yaml

@@ -22,7 +22,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS
               value: "100"
             - name: CHECKPOINTING

theodolite-benchmarks/definitions/uc1-flink/resources/taskmanager-deployment.yaml

@@ -22,7 +22,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS
               value: "100"
             - name: CHECKPOINTING

theodolite-benchmarks/definitions/uc1-hazelcastjet/resources/uc1-hazelcastjet-deployment.yaml

@@ -21,7 +21,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS # Set as default for the applications
               value: "100"
             - name: KUBERNETES_NAMESPACE

theodolite-benchmarks/definitions/uc1-kstreams/resources/uc1-kstreams-deployment.yaml

@@ -23,7 +23,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: JAVA_OPTS
               value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
             - name: COMMIT_INTERVAL_MS # Set as default for the applications

theodolite-benchmarks/definitions/uc1-load-generator/resources/uc1-load-generator-deployment.yaml

@@ -29,4 +29,4 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"

theodolite-benchmarks/definitions/uc2-beam-flink/resources/jobmanager-deployment.yaml

@@ -27,7 +27,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS
               value: "100"
             - name: CHECKPOINTING

theodolite-benchmarks/definitions/uc2-beam-flink/resources/taskmanager-deployment.yaml

@@ -23,7 +23,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS
               value: "100"
             - name: CHECKPOINTING

theodolite-benchmarks/definitions/uc2-beam-samza/resources/uc2-beam-samza-deployment.yaml

@@ -28,7 +28,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS # Set as default for the applications
               value: "100"
           resources:

theodolite-benchmarks/definitions/uc2-flink/resources/jobmanager-deployment.yaml

@@ -22,7 +22,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS
               value: "100"
             - name: CHECKPOINTING

theodolite-benchmarks/definitions/uc2-flink/resources/taskmanager-deployment.yaml

@@ -22,7 +22,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS
               value: "100"
             - name: CHECKPOINTING

theodolite-benchmarks/definitions/uc2-hazelcastjet/resources/uc2-hazelcastjet-deployment.yaml

@@ -21,7 +21,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS # Set as default for the applications
               value: "100"
             - name: DOWNSAMPLE_INTERVAL

theodolite-benchmarks/definitions/uc2-kstreams/resources/uc2-kstreams-deployment.yaml

@@ -23,7 +23,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: JAVA_OPTS
               value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
             - name: COMMIT_INTERVAL_MS # Set as default for the applications

theodolite-benchmarks/definitions/uc2-load-generator/resources/uc2-load-generator-deployment.yaml

@@ -29,4 +29,4 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"

theodolite-benchmarks/definitions/uc3-beam-flink/resources/jobmanager-deployment.yaml

@@ -27,7 +27,7 @@ spec:
             - name: KAFKA_BOOTSTRAP_SERVERS
               value: "theodolite-kafka-kafka-bootstrap:9092"
             - name: SCHEMA_REGISTRY_URL
-              value: "http://theodolite-cp-schema-registry:8081"
+              value: "http://theodolite-kafka-schema-registry:8081"
             - name: COMMIT_INTERVAL_MS
               value: "100"
             - name: CHECKPOINTING