Skip to content
Snippets Groups Projects
Commit 29facf99 authored by Sören Henning's avatar Sören Henning
Browse files

Merge branch 'theodolite-kotlin' into store-results-in-volume

parents cd36cc82 4c185437
No related branches found
No related tags found
4 merge requests!159Re-implementation of Theodolite with Kotlin/Quarkus,!157Update Graal Image in CI pipeline,!153Store benchmark execution results in operator volume,!83WIP: Re-implementation of Theodolite with Kotlin/Quarkus
Pipeline #3700 passed
Showing
with 704 additions and 14 deletions
apiVersion: v1
kind: ConfigMap
metadata:
name: flink-config
labels:
app: flink
data:
flink-conf.yaml: |+
jobmanager.rpc.address: flink-jobmanager
taskmanager.numberOfTaskSlots: 1 #TODO
#blob.server.port: 6124
#jobmanager.rpc.port: 6123
#taskmanager.rpc.port: 6122
#queryable-state.proxy.ports: 6125
#jobmanager.memory.process.size: 4Gb
#taskmanager.memory.process.size: 4Gb
#parallelism.default: 1 #TODO
metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
metrics.reporter.prom.interval: 10 SECONDS
taskmanager.network.detailed-metrics: true
# -> gives metrics about inbound/outbound network queue lengths
log4j-console.properties: |+
# This affects logging for both user code and Flink
rootLogger.level = INFO
rootLogger.appenderRef.console.ref = ConsoleAppender
rootLogger.appenderRef.rolling.ref = RollingFileAppender
# Uncomment this if you want to _only_ change Flink's logging
#logger.flink.name = org.apache.flink
#logger.flink.level = INFO
# The following lines keep the log level of common libraries/connectors on
# log level INFO. The root logger does not override this. You have to manually
# change the log levels here.
logger.akka.name = akka
logger.akka.level = INFO
logger.kafka.name= org.apache.kafka
logger.kafka.level = INFO
logger.hadoop.name = org.apache.hadoop
logger.hadoop.level = INFO
logger.zookeeper.name = org.apache.zookeeper
logger.zookeeper.level = INFO
# Log all infos to the console
appender.console.name = ConsoleAppender
appender.console.type = CONSOLE
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
# Log all infos in the given rolling file
appender.rolling.name = RollingFileAppender
appender.rolling.type = RollingFile
appender.rolling.append = false
appender.rolling.fileName = ${sys:log.file}
appender.rolling.filePattern = ${sys:log.file}.%i
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
appender.rolling.policies.type = Policies
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
appender.rolling.policies.size.size=100MB
appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.max = 10
# Suppress the irrelevant (wrong) warnings from the Netty channel handler
logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
logger.netty.level = OFF
\ No newline at end of file
apiVersion: apps/v1
kind: Deployment
metadata:
name: flink-jobmanager
spec:
replicas: 1
selector:
matchLabels:
app: flink
component: jobmanager
template:
metadata:
labels:
app: flink
component: jobmanager
spec:
containers:
- name: jobmanager
image: ghcr.io/cau-se/theodolite-uc1-flink:latest
env:
- name: KAFKA_BOOTSTRAP_SERVERS
value: "theodolite-cp-kafka:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-cp-schema-registry:8081"
- name: COMMIT_INTERVAL_MS
value: "100"
- name: CHECKPOINTING
value: "false"
- name: PARALLELISM
value: "1"
- name: "FLINK_STATE_BACKEND"
value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- name: FLINK_PROPERTIES
value: |+
blob.server.port: 6124
jobmanager.rpc.port: 6123
taskmanager.rpc.port: 6122
queryable-state.proxy.ports: 6125
jobmanager.memory.process.size: 4Gb
taskmanager.memory.process.size: 4Gb
#parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
cpu: 1000m
args: ["standalone-job", "--job-classname", "theodolite.uc1.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
#command: ['sleep', '60m']
ports:
- containerPort: 6123
name: rpc
- containerPort: 6124
name: blob-server
- containerPort: 8081
name: webui
- containerPort: 9249
name: metrics
livenessProbe:
tcpSocket:
port: 6123
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- name: flink-config-volume-rw
mountPath: /opt/flink/conf
# - name: job-artifacts-volume
# mountPath: /opt/flink/usrlib
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
initContainers:
- name: init-jobmanager
image: busybox:1.28
command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
volumeMounts:
- name: flink-config-volume
mountPath: /flink-config/
- name: flink-config-volume-rw
mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
name: flink-config
items:
- key: flink-conf.yaml
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- name: flink-config-volume-rw
emptyDir: {}
# - name: job-artifacts-volume
# hostPath:
# path: /host/path/to/job/artifacts
\ No newline at end of file
apiVersion: v1
kind: Service
metadata:
name: flink-jobmanager-rest
spec:
type: NodePort
ports:
- name: rest
port: 8081
targetPort: 8081
nodePort: 30081
selector:
app: flink
component: jobmanager
\ No newline at end of file
apiVersion: v1
kind: Service
metadata:
name: flink-jobmanager
labels:
app: flink
spec:
type: ClusterIP
ports:
- name: rpc
port: 6123
- name: blob-server
port: 6124
- name: webui
port: 8081
- name: metrics
port: 9249
selector:
app: flink
component: jobmanager
\ No newline at end of file
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
app: flink
appScope: titan-ccp
name: flink
spec:
selector:
matchLabels:
app: flink
endpoints:
- port: metrics
interval: 10s
apiVersion: apps/v1
kind: Deployment
metadata:
name: flink-taskmanager
spec:
replicas: 1
selector:
matchLabels:
app: flink
component: taskmanager
template:
metadata:
labels:
app: flink
component: taskmanager
spec:
containers:
- name: taskmanager
image: ghcr.io/cau-se/theodolite-uc1-flink:latest
env:
- name: KAFKA_BOOTSTRAP_SERVERS
value: "theodolite-cp-kafka:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-cp-schema-registry:8081"
- name: COMMIT_INTERVAL_MS
value: "100"
- name: CHECKPOINTING
value: "false"
- name: PARALLELISM
value: "1"
- name: "FLINK_STATE_BACKEND"
value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- name: TASK_MANAGER_NUMBER_OF_TASK_SLOTS
value: "1" #TODO
- name: FLINK_PROPERTIES
value: |+
blob.server.port: 6124
jobmanager.rpc.port: 6123
taskmanager.rpc.port: 6122
queryable-state.proxy.ports: 6125
jobmanager.memory.process.size: 4Gb
taskmanager.memory.process.size: 4Gb
#parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
cpu: 1000m
args: ["taskmanager"]
ports:
- containerPort: 6122
name: rpc
- containerPort: 6125
name: query-state
- containerPort: 9249
name: metrics
livenessProbe:
tcpSocket:
port: 6122
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- name: flink-config-volume-rw
mountPath: /opt/flink/conf/
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
initContainers:
- name: init-taskmanager
image: busybox:1.28
command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
volumeMounts:
- name: flink-config-volume
mountPath: /flink-config/
- name: flink-config-volume-rw
mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
name: flink-config
items:
- key: flink-conf.yaml
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- name: flink-config-volume-rw
emptyDir: {}
apiVersion: v1
kind: Service
metadata:
name: flink-taskmanager
labels:
app: flink
spec:
type: ClusterIP
ports:
- name: metrics
port: 9249
selector:
app: flink
component: taskmanager
\ No newline at end of file
......@@ -5,8 +5,8 @@ metadata:
spec:
appResource:
- "uc1-kstreams-deployment.yaml"
- "aggregation-service.yaml"
- "jmx-configmap.yaml"
- "uc1-kstreams-service.yaml"
- "uc1-jmx-configmap.yaml"
- "uc1-service-monitor.yaml"
loadGenResource:
- "uc1-load-generator-deployment.yaml"
......@@ -17,17 +17,17 @@ spec:
- type: "ReplicaPatcher"
resource: "uc1-kstreams-deployment.yaml"
loadTypes:
- typeName: "NumSensors"
patchers:
- type: "EnvVarPatcher"
resource: "uc1-load-generator-deployment.yaml"
properties:
variableName: "NUM_SENSORS"
container: "workload-generator"
- type: "NumSensorsLoadGeneratorReplicaPatcher"
resource: "uc1-load-generator-deployment.yaml"
properties:
loadGenMaxRecords: "15000"
- typeName: "NumSensors"
patchers:
- type: "EnvVarPatcher"
resource: "uc1-load-generator-deployment.yaml"
properties:
container: "workload-generator"
variableName: "NUM_SENSORS"
- type: "NumSensorsLoadGeneratorReplicaPatcher"
resource: "uc1-load-generator-deployment.yaml"
properties:
loadGenMaxRecords: "15000"
kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092"
topics:
......@@ -35,4 +35,4 @@ spec:
numPartitions: 40
replicationFactor: 1
- name: "theodolite-.*"
removeOnly: True
\ No newline at end of file
removeOnly: True
name: "uc1-kstreams"
appResource:
- "uc1-kstreams-deployment.yaml"
- "uc1-kstreams-service.yaml"
- "uc1-jmx-configmap.yaml"
- "uc1-service-monitor.yaml"
loadGenResource:
- "uc1-load-generator-deployment.yaml"
- "uc1-load-generator-service.yaml"
resourceTypes:
- typeName: "Instances"
patchers:
- type: "ReplicaPatcher"
resource: "uc1-kstreams-deployment.yaml"
loadTypes:
- typeName: "NumSensors"
patchers:
- type: "EnvVarPatcher"
resource: "uc1-load-generator-deployment.yaml"
container: "workload-generator"
variableName: "NUM_SENSORS"
- type: NumSensorsLoadGeneratorReplicaPatcher
resource: "uc1-load-generator-deployment.yaml"
properties:
loadGenMaxRecords: "15000"
kafkaConfig:
bootstrapServer: "theodolite-cp-kafka:9092"
topics:
- name: "input"
numPartitions: 40
replicationFactor: 1
\ No newline at end of file
apiVersion: v1
kind: ConfigMap
metadata:
name: aggregation-jmx-configmap
data:
jmx-kafka-prometheus.yml: |+
jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi
lowercaseOutputName: true
lowercaseOutputLabelNames: true
ssl: false
apiVersion: apps/v1
kind: Deployment
metadata:
name: titan-ccp-aggregation
spec:
selector:
matchLabels:
app: titan-ccp-aggregation
replicas: 1
template:
metadata:
labels:
app: titan-ccp-aggregation
spec:
terminationGracePeriodSeconds: 0
containers:
- name: uc-application
image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest
ports:
- containerPort: 5555
name: jmx
env:
- name: KAFKA_BOOTSTRAP_SERVERS
value: "theodolite-cp-kafka:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-cp-schema-registry:8081"
- name: JAVA_OPTS
value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=5555"
- name: COMMIT_INTERVAL_MS # Set as default for the applications
value: "100"
resources:
limits:
memory: 4Gi
cpu: 1000m
- name: prometheus-jmx-exporter
image: "solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143"
command:
- java
- -XX:+UnlockExperimentalVMOptions
- -XX:+UseCGroupMemoryLimitForHeap
- -XX:MaxRAMFraction=1
- -XshowSettings:vm
- -jar
- jmx_prometheus_httpserver.jar
- "5556"
- /etc/jmx-aggregation/jmx-kafka-prometheus.yml
ports:
- containerPort: 5556
volumeMounts:
- name: jmx-config
mountPath: /etc/jmx-aggregation
volumes:
- name: jmx-config
configMap:
name: aggregation-jmx-configmap
\ No newline at end of file
apiVersion: v1
kind: Service
metadata:
name: titan-ccp-aggregation
labels:
app: titan-ccp-aggregation
spec:
#type: NodePort
selector:
app: titan-ccp-aggregation
ports:
- name: http
port: 80
targetPort: 80
protocol: TCP
- name: metrics
port: 5556
apiVersion: apps/v1
kind: Deployment
metadata:
name: titan-ccp-load-generator
spec:
selector:
matchLabels:
app: titan-ccp-load-generator
replicas: 1
template:
metadata:
labels:
app: titan-ccp-load-generator
spec:
terminationGracePeriodSeconds: 0
containers:
- name: workload-generator
image: ghcr.io/cau-se/theodolite-uc1-workload-generator:latest
ports:
- containerPort: 5701
name: coordination
env:
- name: KUBERNETES_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: KUBERNETES_DNS_NAME
value: "titan-ccp-load-generator.$(KUBERNETES_NAMESPACE).svc.cluster.local"
- name: KAFKA_BOOTSTRAP_SERVERS
value: "theodolite-cp-kafka:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-cp-schema-registry:8081"
apiVersion: v1
kind: Service
metadata:
name: titan-ccp-load-generator
labels:
app: titan-ccp-load-generator
spec:
type: ClusterIP
clusterIP: None
selector:
app: titan-ccp-load-generator
ports:
- name: coordination
port: 5701
targetPort: 5701
protocol: TCP
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
app: titan-ccp-aggregation
appScope: titan-ccp
name: titan-ccp-aggregation
spec:
selector:
matchLabels:
app: titan-ccp-aggregation
endpoints:
- port: metrics
interval: 10s
apiVersion: v1
kind: ConfigMap
metadata:
name: flink-config
labels:
app: flink
data:
flink-conf.yaml: |+
#jobmanager.rpc.address: flink-jobmanager
#taskmanager.numberOfTaskSlots: 1 #TODO
#blob.server.port: 6124
#jobmanager.rpc.port: 6123
#taskmanager.rpc.port: 6122
#queryable-state.proxy.ports: 6125
#jobmanager.memory.process.size: 4Gb
#taskmanager.memory.process.size: 4Gb
#parallelism.default: 1 #TODO
metrics.reporter.prom.class: org.apache.flink.metrics.prometheus.PrometheusReporter
metrics.reporter.prom.interval: 10 SECONDS
taskmanager.network.detailed-metrics: true
# -> gives metrics about inbound/outbound network queue lengths
log4j-console.properties: |+
# This affects logging for both user code and Flink
rootLogger.level = INFO
rootLogger.appenderRef.console.ref = ConsoleAppender
rootLogger.appenderRef.rolling.ref = RollingFileAppender
# Uncomment this if you want to _only_ change Flink's logging
#logger.flink.name = org.apache.flink
#logger.flink.level = INFO
# The following lines keep the log level of common libraries/connectors on
# log level INFO. The root logger does not override this. You have to manually
# change the log levels here.
logger.akka.name = akka
logger.akka.level = INFO
logger.kafka.name= org.apache.kafka
logger.kafka.level = INFO
logger.hadoop.name = org.apache.hadoop
logger.hadoop.level = INFO
logger.zookeeper.name = org.apache.zookeeper
logger.zookeeper.level = INFO
# Log all infos to the console
appender.console.name = ConsoleAppender
appender.console.type = CONSOLE
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
# Log all infos in the given rolling file
appender.rolling.name = RollingFileAppender
appender.rolling.type = RollingFile
appender.rolling.append = false
appender.rolling.fileName = ${sys:log.file}
appender.rolling.filePattern = ${sys:log.file}.%i
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
appender.rolling.policies.type = Policies
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
appender.rolling.policies.size.size=100MB
appender.rolling.strategy.type = DefaultRolloverStrategy
appender.rolling.strategy.max = 10
# Suppress the irrelevant (wrong) warnings from the Netty channel handler
logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
logger.netty.level = OFF
\ No newline at end of file
apiVersion: apps/v1
kind: Deployment
metadata:
name: flink-jobmanager
spec:
replicas: 1
selector:
matchLabels:
app: flink
component: jobmanager
template:
metadata:
labels:
app: flink
component: jobmanager
spec:
containers:
- name: jobmanager
image: ghcr.io/cau-se/theodolite-uc2-flink:latest
env:
- name: KAFKA_BOOTSTRAP_SERVERS
value: "theodolite-cp-kafka:9092"
- name: SCHEMA_REGISTRY_URL
value: "http://theodolite-cp-schema-registry:8081"
- name: COMMIT_INTERVAL_MS
value: "100"
- name: CHECKPOINTING
value: "false"
- name: PARALLELISM
value: "1"
- name: "FLINK_STATE_BACKEND"
value: "rocksdb"
- name: JOB_MANAGER_RPC_ADDRESS
value: "flink-jobmanager"
- name: FLINK_PROPERTIES
value: |+
blob.server.port: 6124
jobmanager.rpc.port: 6123
taskmanager.rpc.port: 6122
queryable-state.proxy.ports: 6125
jobmanager.memory.process.size: 4Gb
taskmanager.memory.process.size: 4Gb
#parallelism.default: 1 #TODO
resources:
limits:
memory: 4Gi
cpu: 1000m
args: ["standalone-job", "--job-classname", "theodolite.uc2.application.HistoryServiceFlinkJob"] # optional arguments: ["--job-id", "<job id>", "--fromSavepoint", "/path/to/savepoint", "--allowNonRestoredState"]
#command: ['sleep', '60m']
ports:
- containerPort: 6123
name: rpc
- containerPort: 6124
name: blob-server
- containerPort: 8081
name: webui
- containerPort: 9249
name: metrics
livenessProbe:
tcpSocket:
port: 6123
initialDelaySeconds: 30
periodSeconds: 60
volumeMounts:
- name: flink-config-volume-rw
mountPath: /opt/flink/conf
# - name: job-artifacts-volume
# mountPath: /opt/flink/usrlib
securityContext:
runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary
initContainers:
- name: init-jobmanager
image: busybox:1.28
command: ['cp', '-a', '/flink-config/.', '/flink-config-rw/']
volumeMounts:
- name: flink-config-volume
mountPath: /flink-config/
- name: flink-config-volume-rw
mountPath: /flink-config-rw/
volumes:
- name: flink-config-volume
configMap:
name: flink-config
items:
- key: flink-conf.yaml
path: flink-conf.yaml
- key: log4j-console.properties
path: log4j-console.properties
- name: flink-config-volume-rw
emptyDir: {}
# - name: job-artifacts-volume
# hostPath:
# path: /host/path/to/job/artifacts
\ No newline at end of file
apiVersion: v1
kind: Service
metadata:
name: flink-jobmanager-rest
spec:
type: NodePort
ports:
- name: rest
port: 8081
targetPort: 8081
nodePort: 30081
selector:
app: flink
component: jobmanager
\ No newline at end of file
apiVersion: v1
kind: Service
metadata:
name: flink-jobmanager
labels:
app: flink
spec:
type: ClusterIP
ports:
- name: rpc
port: 6123
- name: blob-server
port: 6124
- name: webui
port: 8081
- name: metrics
port: 9249
selector:
app: flink
component: jobmanager
\ No newline at end of file
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
app: flink
appScope: titan-ccp
name: flink
spec:
selector:
matchLabels:
app: flink
endpoints:
- port: metrics
interval: 10s
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment