diff --git a/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml index 7dc7a2c58cad47efe2dac623a0834648974ea988..20ebf1e0f1b18f9719465c358c5354ceb8f41c8d 100644 --- a/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-beam-flink/uc2-beam-flink-benchmark-operator.yaml @@ -91,6 +91,18 @@ spec: threshold: 2000 externalSloUrl: "http://localhost:80/evaluate-slope" warmup: 60 # in seconds + - name: droppedRecords + sloType: generic + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum(rate(flink_taskmanager_job_task_operator_org_apache_beam_runners_core_LateDataDroppingDoFnRunner_droppedDueToLateness[30s])) by (job) OR clamp_max(absent(notExists{job='absent'}),0)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 100 kafkaConfig: bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: diff --git a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml index b3904ca5f6ecff70ec0d03f77800449784525cb9..0c30c88ffde9cc7de7c4aec33d769edc560ae6c4 100644 --- a/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-flink/uc2-flink-benchmark-operator.yaml @@ -9,16 +9,16 @@ metadata: spec: sut: resources: - - configMap: - name: "benchmark-resources-uc2-flink" - files: - - "flink-configuration-configmap.yaml" - - "taskmanager-deployment.yaml" - - "taskmanager-service.yaml" - - "service-monitor.yaml" - - "jobmanager-service.yaml" - - "jobmanager-deployment.yaml" - # - "jobmanager-rest-service.yaml" + - configMap: + name: "benchmark-resources-uc2-flink" + files: + - "flink-configuration-configmap.yaml" + - "taskmanager-deployment.yaml" + - "taskmanager-service.yaml" + - "service-monitor.yaml" + - "jobmanager-service.yaml" + - "jobmanager-deployment.yaml" + # - "jobmanager-rest-service.yaml" loadGenerator: resources: - configMap: @@ -91,6 +91,18 @@ spec: threshold: 2000 externalSloUrl: "http://localhost:80/evaluate-slope" warmup: 60 # in seconds + - name: droppedRecords + sloType: generic + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum(rate(flink_taskmanager_job_task_operator_numLateRecordsDropped[30s])) by(job)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 100 kafkaConfig: bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: diff --git a/theodolite-benchmarks/definitions/uc2-hazelcastjet/uc2-hazelcastjet-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-hazelcastjet/uc2-hazelcastjet-benchmark-operator.yaml index 101a0e840253654b2c6c0e772ced938ee6c98b7d..419326787e942a0fb69b7e9c9dca48b11903218d 100644 --- a/theodolite-benchmarks/definitions/uc2-hazelcastjet/uc2-hazelcastjet-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-hazelcastjet/uc2-hazelcastjet-benchmark-operator.yaml @@ -66,6 +66,18 @@ spec: threshold: 2000 externalSloUrl: "http://localhost:80/evaluate-slope" warmup: 60 # in seconds + - name: droppedRecords + sloType: generic + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum(rate(com_hazelcast_jet_metrics_lateeventsdropped[30s])) by(job)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 100 kafkaConfig: bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: diff --git a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-operator.yaml index 82f2f8bb8214ea4ae81954ee6af24b0c0b1cded1..3e82c08f33530fc390aeb82cf5e1f7aad0c3090b 100644 --- a/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc2-kstreams/uc2-kstreams-benchmark-operator.yaml @@ -71,6 +71,18 @@ spec: threshold: 2000 externalSloUrl: "http://localhost:80/evaluate-slope" warmup: 60 # in seconds + - name: droppedRecords + sloType: generic + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum(rate(kafka_streams_stream_task_metrics_dropped_records_total[30s])) by (job)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 100 kafkaConfig: bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: diff --git a/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml index 2ef59bd15b08d8d480a1802059aef0402d3e535c..36701522bb2ab671e1873d4c1053851673dab619 100644 --- a/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-beam-flink/uc4-beam-flink-benchmark-operator.yaml @@ -81,7 +81,7 @@ spec: resource: "uc4-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" - numSensors: "4.0" + numSensors: "4" slos: - name: "lag trend" sloType: "lag trend" @@ -92,6 +92,18 @@ spec: threshold: 2000 externalSloUrl: "http://localhost:80/evaluate-slope" warmup: 60 # in seconds + - name: droppedRecords + sloType: generic + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum(rate(flink_taskmanager_job_task_operator_org_apache_beam_runners_core_LateDataDroppingDoFnRunner_droppedDueToLateness[30s])) by (job) OR clamp_max(absent(notExists{job='absent'}),0)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 100 kafkaConfig: bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: diff --git a/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml index 0dd7d9022870acbcd55aa27e049cd40afbc896c4..776b0aeaffcd54437ca5164b8b1f38f0cb60282f 100644 --- a/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-beam-samza/uc4-beam-samza-benchmark-operator.yaml @@ -70,7 +70,7 @@ spec: resource: "uc4-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" - numSensors: "4.0" + numSensors: "4" slos: - name: "lag trend" sloType: "lag trend" diff --git a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml index fec201f77c5a5b6daf8410dcb3ebe9c22f3facbe..4eddd777128b4c1eb7a20ad9013a41d410bf5fe1 100644 --- a/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-flink/uc4-flink-benchmark-operator.yaml @@ -81,7 +81,7 @@ spec: resource: "uc4-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" - numSensors: "4.0" + numSensors: "4" slos: - name: "lag trend" sloType: "lag trend" @@ -92,6 +92,18 @@ spec: threshold: 2000 externalSloUrl: "http://localhost:80/evaluate-slope" warmup: 60 # in seconds + - name: droppedRecords + sloType: generic + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum(rate(flink_taskmanager_job_task_operator_numLateRecordsDropped[30s])) by(job)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 100 kafkaConfig: bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: diff --git a/theodolite-benchmarks/definitions/uc4-hazelcastjet/uc4-hazelcastjet-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-hazelcastjet/uc4-hazelcastjet-benchmark-operator.yaml index 9db8017dffd7cddd257fa99f54ef7a464673c584..2370bbbadbb955a542e3a8171c7beafd6e9b0c75 100644 --- a/theodolite-benchmarks/definitions/uc4-hazelcastjet/uc4-hazelcastjet-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-hazelcastjet/uc4-hazelcastjet-benchmark-operator.yaml @@ -51,12 +51,12 @@ spec: resource: "uc4-load-generator-deployment.yaml" properties: container: "workload-generator" - variableName: "NUM_SENSORS" + variableName: "NUM_NESTED_GROUPS" - type: NumNestedGroupsLoadGeneratorReplicaPatcher resource: "uc4-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" - numSensors: "4.0" + numSensors: "4" slos: - name: "lag trend" sloType: "lag trend" @@ -67,6 +67,18 @@ spec: threshold: 2000 externalSloUrl: "http://localhost:80/evaluate-slope" warmup: 60 # in seconds + - name: droppedRecords + sloType: generic + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum(rate(com_hazelcast_jet_metrics_lateeventsdropped[30s])) by(job)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 100 kafkaConfig: bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: diff --git a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-operator.yaml b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-operator.yaml index 15a388f1af91e83f468b05e6cd1d61a8823e62c6..e43f210f69facf7e3b404f5c1e00592a5e7f549d 100644 --- a/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-operator.yaml +++ b/theodolite-benchmarks/definitions/uc4-kstreams/uc4-kstreams-benchmark-operator.yaml @@ -56,12 +56,12 @@ spec: resource: "uc4-load-generator-deployment.yaml" properties: container: "workload-generator" - variableName: "NUM_SENSORS" + variableName: "NUM_NESTED_GROUPS" - type: NumNestedGroupsLoadGeneratorReplicaPatcher resource: "uc4-load-generator-deployment.yaml" properties: loadGenMaxRecords: "150000" - numSensors: "4.0" + numSensors: "4" slos: - name: "lag trend" sloType: "lag trend" @@ -72,6 +72,18 @@ spec: threshold: 2000 externalSloUrl: "http://localhost:80/evaluate-slope" warmup: 60 # in seconds + - name: droppedRecords + sloType: generic + prometheusUrl: "http://prometheus-operated:9090" + offset: 0 + properties: + externalSloUrl: "http://localhost:8082" + promQLQuery: "sum(rate(kafka_streams_stream_task_metrics_dropped_records_total[30s])) by (job)" + warmup: 60 # in seconds + queryAggregation: max + repetitionAggregation: median + operator: lte + threshold: 100 kafkaConfig: bootstrapServer: "theodolite-kafka-kafka-bootstrap:9092" topics: