diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index daa2cd332cbedd388114f316492f6b4eaa93d307..c97acd1628e7816e70d7791d5702f4dcf4680c42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ workflow: rules: - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push"' + - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_PROTECTED != "true"' when: never - when: always @@ -10,6 +10,7 @@ stages: - test - check - deploy + - smoketest default: tags: @@ -20,9 +21,9 @@ default: - exec-dind # see https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled # for image usage and settings for building with TLS and docker in docker - image: docker:19.03.1 + image: docker:20.10.12 services: - - docker:19.03.1-dind + - docker:20.10.12-dind variables: DOCKER_TLS_CERTDIR: "/certs" @@ -33,12 +34,22 @@ default: script: - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"${CR_HOST}\":{\"auth\":\"$(printf "%s:%s" "${CR_USER}" "${CR_PW}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json - - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') - - "[ ! $CI_COMMIT_TAG ] && KANIKO_D=\"$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest\"" - - "[ ! $CI_COMMIT_TAG ] && KANIKO_D=\"$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA\"" - - "[ $CI_COMMIT_TAG ] && KANIKO_D=\"$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$CI_COMMIT_TAG\"" + - > + if [ $IMAGE_TAG ]; then + KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$IMAGE_TAG" + elif [ $CI_COMMIT_TAG ]; then + KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$CI_COMMIT_TAG" + else + DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//') + KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest" + KANIKO_D="$KANIKO_D -d $CR_HOST/$CR_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA" + fi - "[ $DOCKERFILE ] && KANIKO_DOCKERFILE=\"--dockerfile $DOCKERFILE\"" - /kaniko/executor --context `pwd`/$CONTEXT $KANIKO_DOCKERFILE $KANIKO_D + - echo "PUBLISHED_IMAGE_TAG=${CI_COMMIT_TAG-$CI_COMMIT_SHORT_SHA}" >> build.env + artifacts: + reports: + dotenv: build.env # Theodolite Docs @@ -73,26 +84,37 @@ test-docs-links: - build-docs script: bundle exec htmlproofer --assume-extension --allow_hash_href ./_site +build-docs-crds: + stage: build + image: + name: ghcr.io/fybrik/crdoc:0.6.1 + entrypoint: [""] + script: /crdoc --resources theodolite/crd/ --template docs/api-reference/crds.tmpl --output docs/api-reference/crds.ref.md + artifacts: + paths: + - docs/api-reference/crds.ref.md + expire_in: 1 week + rules: + - changes: + - docs/api-reference/crds.tmpl + - theodolite/crd/**/* + - when: manual + allow_failure: true + test-docs-crds-regression: stage: test - image: golang + needs: + - build-docs-crds + image: alpine:3.15 before_script: - cd docs - - go install fybrik.io/crdoc@latest script: - - crdoc --resources ../theodolite/crd/ --template api-reference/crds.tmpl --output api-reference/crds.ref.md - cmp api-reference/crds.md api-reference/crds.ref.md artifacts: when: on_failure paths: - docs/api-reference/crds.ref.md expire_in: 1 week - rules: - - changes: - - docs/api-reference/crds.tmpl - - theodolite/crd/**/* - - when: manual - allow_failure: true # Theodolite Helm Chart @@ -104,6 +126,11 @@ lint-helm: name: alpine/helm:3.5.2 entrypoint: [""] script: helm lint helm/ + rules: + - changes: + - helm/* + - when: manual + allow_failure: true # Theodolite Benchmarks @@ -350,7 +377,174 @@ deploy-uc4-load-generator: IMAGE_NAME: "theodolite-uc4-workload-generator" JAVA_PROJECT_NAME: "uc4-load-generator" JAVA_PROJECT_DEPS: "load-generator-commons" - + +deploy-http-bridge: + extends: .deploy-benchmarks + variables: + IMAGE_NAME: "theodolite-http-bridge" + JAVA_PROJECT_NAME: "http-bridge" + JAVA_PROJECT_DEPS: "load-generator-commons" + +.smoketest-benchmarks: + stage: smoketest + extends: + - .dind + image: ghcr.io/cau-se/theodolite-build-docker-compose-jq:20.10.12 + before_script: + - cd theodolite-benchmarks/docker-test + # variables: + # TEST_LOG_FILE: "test.log" + script: + - export THEODOLITE_TAG=$PUBLISHED_IMAGE_TAG + - ./smoketest-runner.sh ./$DOCKER_COMPOSE_DIR + # - cat test.log + after_script: + - cd ./$DOCKER_COMPOSE_DIR + - docker-compose down + rules: + - changes: + - theodolite-benchmarks/* + - theodolite-benchmarks/{$JAVA_PROJECT_DEPS}/**/* + if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS" + - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $DOCKER_COMPOSE_DIR && $JAVA_PROJECT_DEPS" + when: manual + allow_failure: true + +smoketest-uc1-kstreams: + extends: .smoketest-benchmarks + needs: + - deploy-uc1-kstreams + - deploy-uc1-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc1-kstreams" + JAVA_PROJECT_DEPS: "uc1-kstreams,kstreams-commons,uc1-load-generator,load-generator-commons" + +smoketest-uc1-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc1-flink + - deploy-uc1-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc1-flink" + JAVA_PROJECT_DEPS: "uc1-flink,flink-commons,uc1-load-generator,load-generator-commons" + +smoketest-uc1-beam-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc1-beam-flink + - deploy-uc1-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc1-beam-flink" + JAVA_PROJECT_DEPS: "uc1-beam-flink,uc1-beam,beam-commons,uc1-load-generator,load-generator-commons" + +smoketest-uc1-beam-samza: + extends: .smoketest-benchmarks + needs: + - deploy-uc1-beam-samza + - deploy-uc1-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc1-beam-samza" + JAVA_PROJECT_DEPS: "uc1-beam-samza,uc1-beam,beam-commons,uc1-load-generator,load-generator-commons" + +smoketest-uc2-kstreams: + extends: .smoketest-benchmarks + needs: + - deploy-uc2-kstreams + - deploy-uc2-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc2-kstreams" + JAVA_PROJECT_DEPS: "uc2-kstreams,kstreams-commons,uc2-load-generator,load-generator-commons" + +smoketest-uc2-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc2-flink + - deploy-uc2-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc2-flink" + JAVA_PROJECT_DEPS: "uc2-flink,flink-commons,uc2-load-generator,load-generator-commons" + +smoketest-uc2-beam-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc2-beam-flink + - deploy-uc2-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc2-beam-flink" + JAVA_PROJECT_DEPS: "uc2-beam-flink,uc2-beam,beam-commons,uc2-load-generator,load-generator-commons" + +smoketest-uc2-beam-samza: + extends: .smoketest-benchmarks + needs: + - deploy-uc2-beam-samza + - deploy-uc2-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc2-beam-samza" + JAVA_PROJECT_DEPS: "uc2-beam-samza,uc2-beam,beam-commons,uc2-load-generator,load-generator-commons" + +smoketest-uc3-kstreams: + extends: .smoketest-benchmarks + needs: + - deploy-uc3-kstreams + - deploy-uc3-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc3-kstreams" + JAVA_PROJECT_DEPS: "uc3-kstreams,kstreams-commons,uc3-load-generator,load-generator-commons" + +smoketest-uc3-beam-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc3-beam-flink + - deploy-uc3-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc3-beam-flink" + JAVA_PROJECT_DEPS: "uc3-beam-flink,uc3-beam,beam-commons,uc3-load-generator,load-generator-commons" + +smoketest-uc3-beam-samza: + extends: .smoketest-benchmarks + needs: + - deploy-uc3-beam-samza + - deploy-uc3-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc3-beam-samza" + JAVA_PROJECT_DEPS: "uc3-beam-samza,uc3-beam,beam-commons,uc3-load-generator,load-generator-commons" + +smoketest-uc4-kstreams: + extends: .smoketest-benchmarks + needs: + - deploy-uc4-kstreams + - deploy-uc4-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc4-kstreams" + JAVA_PROJECT_DEPS: "uc4-kstreams,kstreams-commons,uc4-load-generator,load-generator-commons" + +smoketest-uc4-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc4-flink + - deploy-uc4-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc4-flink" + JAVA_PROJECT_DEPS: "uc4-flink,flink-commons,uc4-load-generator,load-generator-commons" + +smoketest-uc4-beam-flink: + extends: .smoketest-benchmarks + needs: + - deploy-uc4-beam-flink + - deploy-uc4-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc4-beam-flink" + JAVA_PROJECT_DEPS: "uc4-beam-flink,uc4-beam,beam-commons,uc4-load-generator,load-generator-commons" + +smoketest-uc4-beam-samza: + extends: .smoketest-benchmarks + needs: + - deploy-uc4-beam-samza + - deploy-uc4-load-generator + variables: + DOCKER_COMPOSE_DIR: "uc4-beam-samza" + JAVA_PROJECT_DEPS: "uc4-beam-samza,uc4-beam,beam-commons,uc4-load-generator,load-generator-commons" + # Theodolite Framework @@ -367,6 +561,11 @@ deploy-uc4-load-generator: before_script: - export GRADLE_USER_HOME=`pwd`/.gradle - cd theodolite + rules: + - changes: + - theodolite/**/* + - when: manual + allow_failure: true build-theodolite-jvm: stage: build @@ -567,4 +766,22 @@ deploy-random-scheduler: - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" when: manual allow_failure: true - \ No newline at end of file + +deploy-buildimage-docker-compose-jq: + stage: deploy + extends: + - .kaniko-push + needs: [] + variables: + DOCKER_VERSION: 20.10.12 + IMAGE_NAME: theodolite-build-docker-compose-jq + IMAGE_TAG: $DOCKER_VERSION + before_script: + - cd buildimages/docker-compose-jq + rules: + - changes: + - buildimages/docker-compose-jq/Dockerfile + if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW" + - if: "$CR_HOST && $CR_ORG && $CR_USER && $CR_PW && $CI_PIPELINE_SOURCE == 'web'" + when: manual + allow_failure: true diff --git a/buildimages/docker-compose-jq/Dockerfile b/buildimages/docker-compose-jq/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..cd3f60ba3e75ab5767acff788c3bb69c8640cd4c --- /dev/null +++ b/buildimages/docker-compose-jq/Dockerfile @@ -0,0 +1,6 @@ +FROM docker:${DOCKER_VERSION:-latest} + +RUN apk update && \ + apk add jq && \ + apk add py-pip python3-dev libffi-dev openssl-dev gcc libc-dev rust cargo make && \ + pip install docker-compose diff --git a/docs/README.md b/docs/README.md index 52b5311295e5a96721d9aa42f7e9c319da06960c..a19f94305dfdcb1de7c46da98afbb52b28a6bfa0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -39,5 +39,5 @@ crdoc --resources ../theodolite/crd/ --template api-reference/crds.tmpl --outpu With the following command, crdoc is executed in Docker: ```sh -docker run --rm -v "`pwd`/../theodolite/crd/":/crd -u $UID -v "`pwd`/api-reference":/api-reference ghcr.io/fybrik/crdoc:0.6.0 --resources /crd/ --template /api-reference/crds.tmpl --output /api-reference/crds.md +docker run --rm -v "`pwd`/../theodolite/crd/":/crd -v "`pwd`/api-reference":/api-reference ghcr.io/fybrik/crdoc:0.6.1 --resources /crd/ --template /api-reference/crds.tmpl --output /api-reference/crds.md ``` diff --git a/docs/api-reference/patchers.md b/docs/api-reference/patchers.md index 77f937e38f6a0ee4084cb0ad5b5838718eabff10..bea63ccd23decef5654f257221ce0358b4f68e45 100644 --- a/docs/api-reference/patchers.md +++ b/docs/api-reference/patchers.md @@ -53,6 +53,27 @@ Patchers can be seen as functions which take a value as input and modify a Kuber * **resource**: "uc1-kstreams-deployment.yaml" * **example value**: "random-scheduler" +* **LabelPatcher**: Changes the label of a Kubernetes Deployment or StatefulSet. The patched field is: `metadata.labels` + * **type**: "LabelPatcher" + * **resource**: "uc1-kstreams-deployment.yaml" + * **properties**: + * variableName: "app" + * **example value**: "theodolite-sut" + +* **MatchLabelPatcher**: Changes the match labels of a Kubernetes Deployment or StatefulSet. The patched field is: `spec.selector.matchLabels` + * **type**: "MatchLabelPatcher" + * **resource**: "uc1-kstreams-deployment.yaml" + * **properties**: + * variableName: "app" + * **example value**: "theodolite-sut" + +* **TemplateLabelPatcher**: Changes the template labels of a Kubernetes Deployment or StatefulSet. The patched field is: `spec.template.metadata.labels` + * **type**: "MatchLabelPatcher" + * **resource**: "uc1-kstreams-deployment.yaml" + * **properties**: + * variableName: "app" + * **example value**: "theodolite-sut" + * **ImagePatcher**: Changes the image of a Kubernetes resource. **Currently not fully implemented.** * **type**: "ImagePatcher" * **resource**: "uc1-kstreams-deployment.yaml" diff --git a/docs/creating-a-benchmark.md b/docs/creating-a-benchmark.md index 2b1d93bbec0afb3a8897a77439388538fc75c4a6..fde8ba0759407ddea8befc18e244784a9ba34c1f 100644 --- a/docs/creating-a-benchmark.md +++ b/docs/creating-a-benchmark.md @@ -115,15 +115,13 @@ If a benchmark is [executed by an Execution](running-benchmarks), these patchers ## Kafka Configuration -Theodolite allows to automatically create and remove Kafka topics for each SLO experiment. -Use the `removeOnly: True` property for topics which are created automatically by the SUT. -For those topics, also wildcards are allowed in the topic name. +Theodolite allows to automatically create and remove Kafka topics for each SLO experiment by setting a `kafkaConfig`. +It `bootstrapServer` needs to point your Kafka cluster and `topics` configures the list of Kafka topics to be created/removed. +For each topic, you configure its name, the number of partitions and the replication factor. -If no Kafka topics should be created, simply set: - -```yaml -kafkaConfig: [] -``` +With the `removeOnly: True` property, you can also instruct Theodolite to only remove topics and not create them. +This is useful when benchmarking SUTs, which create topics on their own (e.g., Kafka Streams and Samza applications). +For those topics, also wildcards are allowed in the topic name and, of course, no partition count or replication factor must be provided. <!-- Further information: API Reference --> diff --git a/docs/theodolite-benchmarks/load-generator.md b/docs/theodolite-benchmarks/load-generator.md index 6d42ea06d9cb008a9aeddcc8145a2868c8d916b1..17845c42d47e94a5b696dee1d774890de8d6fff1 100644 --- a/docs/theodolite-benchmarks/load-generator.md +++ b/docs/theodolite-benchmarks/load-generator.md @@ -17,7 +17,7 @@ For each benchmark, we provide a [load generator as OCI container image](https:/ You can simply run a load generator container, for example, for benchmark UC1 with: ```sh -docker run ghcr.io/cau-se/theodolite-uc1-workload-generator +docker run -it ghcr.io/cau-se/theodolite-uc1-workload-generator ``` ### Message format diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 27451ad55ce75592db9dc7550b1f81dced3951bc..52fda2bbc83c722d70f6179c4ec53fb3775bd436 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -26,7 +26,7 @@ dependencies: condition: cp-helm-charts.enabled - name: kafka-lag-exporter version: 0.6.7 - repository: https://lightbend.github.io/kafka-lag-exporter/repo/ + repository: https://seanglover.com/kafka-lag-exporter/repo condition: kafka-lag-exporter.enabled version: 0.7.0-SNAPSHOT diff --git a/theodolite-benchmarks/beam-commons/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/beam-commons/.settings/org.eclipse.jdt.ui.prefs new file mode 100644 index 0000000000000000000000000000000000000000..60b9977149c7b281cb2ac91ee282f73d4351e348 --- /dev/null +++ b/theodolite-benchmarks/beam-commons/.settings/org.eclipse.jdt.ui.prefs @@ -0,0 +1,127 @@ +cleanup.add_default_serial_version_id=true +cleanup.add_generated_serial_version_id=false +cleanup.add_missing_annotations=true +cleanup.add_missing_deprecated_annotations=true +cleanup.add_missing_methods=false +cleanup.add_missing_nls_tags=false +cleanup.add_missing_override_annotations=true +cleanup.add_missing_override_annotations_interface_methods=true +cleanup.add_serial_version_id=false +cleanup.always_use_blocks=true +cleanup.always_use_parentheses_in_expressions=false +cleanup.always_use_this_for_non_static_field_access=true +cleanup.always_use_this_for_non_static_method_access=true +cleanup.convert_functional_interfaces=false +cleanup.convert_to_enhanced_for_loop=true +cleanup.correct_indentation=true +cleanup.format_source_code=true +cleanup.format_source_code_changes_only=false +cleanup.insert_inferred_type_arguments=false +cleanup.make_local_variable_final=true +cleanup.make_parameters_final=true +cleanup.make_private_fields_final=true +cleanup.make_type_abstract_if_missing_method=false +cleanup.make_variable_declarations_final=true +cleanup.never_use_blocks=false +cleanup.never_use_parentheses_in_expressions=true +cleanup.organize_imports=true +cleanup.qualify_static_field_accesses_with_declaring_class=false +cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +cleanup.qualify_static_member_accesses_with_declaring_class=true +cleanup.qualify_static_method_accesses_with_declaring_class=false +cleanup.remove_private_constructors=true +cleanup.remove_redundant_modifiers=false +cleanup.remove_redundant_semicolons=true +cleanup.remove_redundant_type_arguments=true +cleanup.remove_trailing_whitespaces=true +cleanup.remove_trailing_whitespaces_all=true +cleanup.remove_trailing_whitespaces_ignore_empty=false +cleanup.remove_unnecessary_casts=true +cleanup.remove_unnecessary_nls_tags=true +cleanup.remove_unused_imports=true +cleanup.remove_unused_local_variables=false +cleanup.remove_unused_private_fields=true +cleanup.remove_unused_private_members=false +cleanup.remove_unused_private_methods=true +cleanup.remove_unused_private_types=true +cleanup.sort_members=false +cleanup.sort_members_all=false +cleanup.use_anonymous_class_creation=false +cleanup.use_blocks=true +cleanup.use_blocks_only_for_return_and_throw=false +cleanup.use_lambda=true +cleanup.use_parentheses_in_expressions=true +cleanup.use_this_for_non_static_field_access=true +cleanup.use_this_for_non_static_field_access_only_if_necessary=false +cleanup.use_this_for_non_static_method_access=true +cleanup.use_this_for_non_static_method_access_only_if_necessary=false +cleanup_profile=_CAU-SE-Style +cleanup_settings_version=2 +eclipse.preferences.version=1 +editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true +formatter_profile=_CAU-SE-Style +formatter_settings_version=21 +org.eclipse.jdt.ui.ignorelowercasenames=true +org.eclipse.jdt.ui.importorder=; +org.eclipse.jdt.ui.ondemandthreshold=99 +org.eclipse.jdt.ui.staticondemandthreshold=99 +sp_cleanup.add_default_serial_version_id=true +sp_cleanup.add_generated_serial_version_id=false +sp_cleanup.add_missing_annotations=true +sp_cleanup.add_missing_deprecated_annotations=true +sp_cleanup.add_missing_methods=false +sp_cleanup.add_missing_nls_tags=false +sp_cleanup.add_missing_override_annotations=true +sp_cleanup.add_missing_override_annotations_interface_methods=true +sp_cleanup.add_serial_version_id=false +sp_cleanup.always_use_blocks=true +sp_cleanup.always_use_parentheses_in_expressions=false +sp_cleanup.always_use_this_for_non_static_field_access=true +sp_cleanup.always_use_this_for_non_static_method_access=true +sp_cleanup.convert_functional_interfaces=false +sp_cleanup.convert_to_enhanced_for_loop=true +sp_cleanup.correct_indentation=true +sp_cleanup.format_source_code=true +sp_cleanup.format_source_code_changes_only=false +sp_cleanup.insert_inferred_type_arguments=false +sp_cleanup.make_local_variable_final=true +sp_cleanup.make_parameters_final=true +sp_cleanup.make_private_fields_final=true +sp_cleanup.make_type_abstract_if_missing_method=false +sp_cleanup.make_variable_declarations_final=true +sp_cleanup.never_use_blocks=false +sp_cleanup.never_use_parentheses_in_expressions=true +sp_cleanup.on_save_use_additional_actions=true +sp_cleanup.organize_imports=true +sp_cleanup.qualify_static_field_accesses_with_declaring_class=false +sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_with_declaring_class=true +sp_cleanup.qualify_static_method_accesses_with_declaring_class=false +sp_cleanup.remove_private_constructors=true +sp_cleanup.remove_redundant_modifiers=false +sp_cleanup.remove_redundant_semicolons=true +sp_cleanup.remove_redundant_type_arguments=true +sp_cleanup.remove_trailing_whitespaces=true +sp_cleanup.remove_trailing_whitespaces_all=true +sp_cleanup.remove_trailing_whitespaces_ignore_empty=false +sp_cleanup.remove_unnecessary_casts=true +sp_cleanup.remove_unnecessary_nls_tags=true +sp_cleanup.remove_unused_imports=true +sp_cleanup.remove_unused_local_variables=false +sp_cleanup.remove_unused_private_fields=true +sp_cleanup.remove_unused_private_members=false +sp_cleanup.remove_unused_private_methods=true +sp_cleanup.remove_unused_private_types=true +sp_cleanup.sort_members=false +sp_cleanup.sort_members_all=false +sp_cleanup.use_anonymous_class_creation=false +sp_cleanup.use_blocks=true +sp_cleanup.use_blocks_only_for_return_and_throw=false +sp_cleanup.use_lambda=true +sp_cleanup.use_parentheses_in_expressions=true +sp_cleanup.use_this_for_non_static_field_access=true +sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=false +sp_cleanup.use_this_for_non_static_method_access=true +sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=false \ No newline at end of file diff --git a/theodolite-benchmarks/beam-commons/build.gradle b/theodolite-benchmarks/beam-commons/build.gradle index a809f6bc4b97d8d62b807243eddecda8a5de5032..64ac2bb51ae1e6d741749a81e5c6c9e296d14d68 100644 --- a/theodolite-benchmarks/beam-commons/build.gradle +++ b/theodolite-benchmarks/beam-commons/build.gradle @@ -13,21 +13,19 @@ repositories { } dependencies { - // These dependencies are used internally, and not exposed to consumers on their own compile classpath. implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true } implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true } - implementation 'com.google.code.gson:gson:2.8.2' - implementation 'com.google.guava:guava:24.1-jre' - implementation('org.apache.beam:beam-sdks-java-io-kafka:2.22.0'){ + implementation group: 'org.apache.beam', name: 'beam-sdks-java-core', version: '2.35.0' + implementation('org.apache.beam:beam-sdks-java-io-kafka:2.35.0'){ exclude group: 'org.apache.kafka', module: 'kafka-clients' } + implementation ('io.confluent:kafka-streams-avro-serde:5.3.2') + implementation group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.30' - implementation group: 'org.apache.beam', name: 'beam-sdks-java-core', version: '2.22.0' runtimeOnly 'org.slf4j:slf4j-api:1.7.32' runtimeOnly 'org.slf4j:slf4j-jdk14:1.7.32' - // Use JUnit test framework testImplementation 'junit:junit:4.12' } diff --git a/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/AbstractPipeline.java b/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/AbstractPipeline.java index c936ce918c10f3c500cdd26f7e057cd7b6c555b6..3f04bf4373aab0394ff4574b4020065ac356724b 100644 --- a/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/AbstractPipeline.java +++ b/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/AbstractPipeline.java @@ -12,6 +12,9 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; */ public class AbstractPipeline extends Pipeline { + private static final String KAFKA_CONFIG_SPECIFIC_AVRO_READER = "specific.avro.reader"; // NOPMD + private static final String KAFKA_CONFIG_SCHEMA_REGISTRY_URL = "schema.registry.url"; // NOPMD + protected final String inputTopic; protected final String bootstrapServer; // Application Configurations @@ -21,8 +24,8 @@ public class AbstractPipeline extends Pipeline { super(options); this.config = config; - inputTopic = config.getString(ConfigurationKeys.KAFKA_INPUT_TOPIC); - bootstrapServer = config.getString(ConfigurationKeys.KAFKA_BOOTSTRAP_SERVERS); + this.inputTopic = config.getString(ConfigurationKeys.KAFKA_INPUT_TOPIC); + this.bootstrapServer = config.getString(ConfigurationKeys.KAFKA_BOOTSTRAP_SERVERS); } /** @@ -32,19 +35,37 @@ public class AbstractPipeline extends Pipeline { */ public Map<String, Object> buildConsumerConfig() { final Map<String, Object> consumerConfig = new HashMap<>(); - consumerConfig.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, - config.getString(ConfigurationKeys.ENABLE_AUTO_COMMIT_CONFIG)); - consumerConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, - config - .getString(ConfigurationKeys.AUTO_OFFSET_RESET_CONFIG)); - consumerConfig.put("schema.registry.url", - config.getString(ConfigurationKeys.SCHEMA_REGISTRY_URL)); - - consumerConfig.put("specific.avro.reader", - config.getString(ConfigurationKeys.SPECIFIC_AVRO_READER)); - - final String applicationName = config.getString(ConfigurationKeys.APPLICATION_NAME); - consumerConfig.put(ConsumerConfig.GROUP_ID_CONFIG, applicationName); + consumerConfig.put( + ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, + this.config.getString(ConfigurationKeys.ENABLE_AUTO_COMMIT_CONFIG)); + consumerConfig.put( + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, + this.config.getString(ConfigurationKeys.AUTO_OFFSET_RESET_CONFIG)); + consumerConfig.put( + KAFKA_CONFIG_SCHEMA_REGISTRY_URL, + this.config.getString(ConfigurationKeys.SCHEMA_REGISTRY_URL)); + consumerConfig.put( + KAFKA_CONFIG_SPECIFIC_AVRO_READER, + this.config.getString(ConfigurationKeys.SPECIFIC_AVRO_READER)); + consumerConfig.put( + ConsumerConfig.GROUP_ID_CONFIG, + this.config.getString(ConfigurationKeys.APPLICATION_NAME)); return consumerConfig; } + + /** + * Builds a simple configuration for a Kafka producer transformation. + * + * @return the build configuration. + */ + public Map<String, Object> buildProducerConfig() { + final Map<String, Object> config = new HashMap<>(); + config.put( + KAFKA_CONFIG_SCHEMA_REGISTRY_URL, + this.config.getString(ConfigurationKeys.SCHEMA_REGISTRY_URL)); + config.put( + KAFKA_CONFIG_SPECIFIC_AVRO_READER, + this.config.getString(ConfigurationKeys.SPECIFIC_AVRO_READER)); + return config; + } } diff --git a/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/ActivePowerRecordDeserializer.java b/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/ActivePowerRecordDeserializer.java new file mode 100644 index 0000000000000000000000000000000000000000..c53dde3d5f4b7d18822c916a637c356b898fe2cd --- /dev/null +++ b/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/ActivePowerRecordDeserializer.java @@ -0,0 +1,11 @@ +package theodolite.commons.beam.kafka; + +import io.confluent.kafka.streams.serdes.avro.SpecificAvroDeserializer; +import org.apache.kafka.common.serialization.Deserializer; +import titan.ccp.model.records.ActivePowerRecord; + +/** + * A Kafka {@link Deserializer} for typed Schema Registry {@link ActivePowerRecord}. + */ +public class ActivePowerRecordDeserializer extends SpecificAvroDeserializer<ActivePowerRecord> { +} diff --git a/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaActivePowerRecordReader.java b/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaActivePowerRecordReader.java deleted file mode 100644 index f102bee41d66c251ecb66418dd3b90dced32cffb..0000000000000000000000000000000000000000 --- a/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaActivePowerRecordReader.java +++ /dev/null @@ -1,61 +0,0 @@ -package theodolite.commons.beam.kafka; - -import io.confluent.kafka.serializers.KafkaAvroDeserializer; -import java.util.Map; -import org.apache.beam.sdk.coders.AvroCoder; -import org.apache.beam.sdk.io.kafka.KafkaIO; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PBegin; -import org.apache.beam.sdk.values.PCollection; -import org.apache.kafka.common.serialization.StringDeserializer; -import titan.ccp.model.records.ActivePowerRecord; - -/** - * Simple {@link PTransform} that read from Kafka using {@link KafkaIO}. - */ -public class KafkaActivePowerRecordReader extends - PTransform<PBegin, PCollection<KV<String, ActivePowerRecord>>> { - - private static final long serialVersionUID = 2603286150183186115L; - private final PTransform<PBegin, PCollection<KV<String, ActivePowerRecord>>> reader; - - - /** - * Instantiates a {@link PTransform} that reads from Kafka with the given Configuration. - */ - public KafkaActivePowerRecordReader(final String bootstrapServer, final String inputTopic, - final Map<String, Object> consumerConfig) { - super(); - - if (bootstrapServer == null) { - throw new IllegalArgumentException("bootstrapServer is null"); - } - - if (inputTopic == null) { - throw new IllegalArgumentException("inputTopic is null"); - } - - // Check if boostrap server and inputTopic are defined - if (bootstrapServer.isEmpty() || inputTopic.isEmpty()) { - throw new IllegalArgumentException("bootstrapServer or inputTopic missing"); - } - - - reader = - KafkaIO.<String, ActivePowerRecord>read() - .withBootstrapServers(bootstrapServer) - .withTopic(inputTopic) - .withKeyDeserializer(StringDeserializer.class) - .withValueDeserializerAndCoder((Class) KafkaAvroDeserializer.class, - AvroCoder.of(ActivePowerRecord.class)) - .withConsumerConfigUpdates(consumerConfig) - .withoutMetadata(); - } - - @Override - public PCollection<KV<String, ActivePowerRecord>> expand(final PBegin input) { - return input.apply(this.reader); - } - -} diff --git a/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaActivePowerTimestampReader.java b/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaActivePowerTimestampReader.java index 732afe9a0c1d4bdfea876025fceea0c5da1310fe..7a48bd71d497f65351888425d092decf5adb05f3 100644 --- a/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaActivePowerTimestampReader.java +++ b/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaActivePowerTimestampReader.java @@ -1,6 +1,5 @@ package theodolite.commons.beam.kafka; -import io.confluent.kafka.serializers.KafkaAvroDeserializer; import java.util.Map; import org.apache.beam.sdk.coders.AvroCoder; import org.apache.beam.sdk.io.kafka.KafkaIO; @@ -12,40 +11,37 @@ import org.apache.kafka.common.serialization.StringDeserializer; import titan.ccp.model.records.ActivePowerRecord; /** - * Simple {@link PTransform} that read from Kafka using {@link KafkaIO}. - * Has additional a TimestampPolicy. + * Simple {@link PTransform} that reads from Kafka using {@link KafkaIO} with event time. */ -public class KafkaActivePowerTimestampReader extends - PTransform<PBegin, PCollection<KV<String, ActivePowerRecord>>> { +public class KafkaActivePowerTimestampReader + extends PTransform<PBegin, PCollection<KV<String, ActivePowerRecord>>> { private static final long serialVersionUID = 2603286150183186115L; private final PTransform<PBegin, PCollection<KV<String, ActivePowerRecord>>> reader; - /** * Instantiates a {@link PTransform} that reads from Kafka with the given Configuration. */ - public KafkaActivePowerTimestampReader(final String bootstrapServer, final String inputTopic, - final Map<String, Object> consumerConfig) { + public KafkaActivePowerTimestampReader( + final String bootstrapServer, + final String inputTopic, + final Map<String, Object> consumerConfig) { super(); - // Check if boostrap server and inputTopic are defined + // Check if bootstrap server and inputTopic are defined if (bootstrapServer.isEmpty() || inputTopic.isEmpty()) { throw new IllegalArgumentException("bootstrapServer or inputTopic missing"); } - reader = - KafkaIO.<String, ActivePowerRecord>read() - .withBootstrapServers(bootstrapServer) - .withTopic(inputTopic) - .withKeyDeserializer(StringDeserializer.class) - .withValueDeserializerAndCoder((Class) KafkaAvroDeserializer.class, - AvroCoder.of(ActivePowerRecord.class)) - .withConsumerConfigUpdates(consumerConfig) - // Set TimeStampPolicy for event time - .withTimestampPolicyFactory( - (tp, previousWaterMark) -> new EventTimePolicy(previousWaterMark)) - .withoutMetadata(); + this.reader = KafkaIO.<String, ActivePowerRecord>read().withBootstrapServers(bootstrapServer) + .withTopic(inputTopic).withKeyDeserializer(StringDeserializer.class) + .withValueDeserializerAndCoder( + ActivePowerRecordDeserializer.class, + AvroCoder.of(ActivePowerRecord.class)) + .withConsumerConfigUpdates(consumerConfig) + .withTimestampPolicyFactory( + (tp, previousWatermark) -> new EventTimePolicy(previousWatermark)) + .withoutMetadata(); } @Override diff --git a/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaWriterTransformation.java b/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaWriterTransformation.java index 0a3867e71479e36ce30a9f222dfd0a7d473bd209..6d33f6f01493c10a1eb6aca56dd309ae58ce4b8d 100644 --- a/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaWriterTransformation.java +++ b/theodolite-benchmarks/beam-commons/src/main/java/theodolite/commons/beam/kafka/KafkaWriterTransformation.java @@ -1,5 +1,6 @@ package theodolite.commons.beam.kafka; +import java.util.Map; import org.apache.beam.sdk.io.kafka.KafkaIO; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.KV; @@ -9,23 +10,35 @@ import org.apache.kafka.common.serialization.Serializer; import org.apache.kafka.common.serialization.StringSerializer; /** - * Wrapper for a Kafka writing Transformation - * where the value type can be generic. + * Wrapper for a Kafka writing Transformation where the value type can be generic. + * * @param <T> type of the value. */ -public class KafkaWriterTransformation<T> extends - PTransform<PCollection<KV<String, T>>, PDone> { +public class KafkaWriterTransformation<T> extends PTransform<PCollection<KV<String, T>>, PDone> { private static final long serialVersionUID = 3171423303843174723L; private final PTransform<PCollection<KV<String, T>>, PDone> writer; /** - * Creates a new kafka writer transformation. + * Creates a new Kafka writer transformation. */ - public KafkaWriterTransformation(final String bootstrapServer, final String outputTopic, - final Class<? extends Serializer<T>> valueSerializer) { + public KafkaWriterTransformation( + final String bootstrapServer, + final String outputTopic, + final Class<? extends Serializer<T>> valueSerializer) { + this(bootstrapServer, outputTopic, valueSerializer, Map.of()); + } + + /** + * Creates a new Kafka writer transformation. + */ + public KafkaWriterTransformation( + final String bootstrapServer, + final String outputTopic, + final Class<? extends Serializer<T>> valueSerializer, + final Map<String, Object> producerConfig) { super(); - // Check if boostrap server and outputTopic are defined + // Check if bootstrap server and outputTopic are defined if (bootstrapServer.isEmpty() || outputTopic.isEmpty()) { throw new IllegalArgumentException("bootstrapServer or outputTopic missing"); } @@ -34,7 +47,8 @@ public class KafkaWriterTransformation<T> extends .withBootstrapServers(bootstrapServer) .withTopic(outputTopic) .withKeySerializer(StringSerializer.class) - .withValueSerializer(valueSerializer); + .withValueSerializer(valueSerializer) + .withProducerConfigUpdates(producerConfig); } diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.flink.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.flink.gradle index eb9bda1f84c4f20568fca1498462dff9082ea1fa..02bf925f8dc29456f9eeaddeef800d2edcf837f8 100644 --- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.flink.gradle +++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.flink.gradle @@ -3,6 +3,6 @@ plugins { } dependencies { - implementation group: 'org.apache.beam', name: 'beam-runners-flink-1.12', version: '2.27.0' - implementation group: 'org.apache.flink', name: 'flink-statebackend-rocksdb_2.11', version: '1.12.0' + implementation group: 'org.apache.beam', name: 'beam-runners-flink-1.13', version: '2.35.0' + implementation group: 'org.apache.flink', name: 'flink-statebackend-rocksdb_2.11', version: '1.13.0' } \ No newline at end of file diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.gradle index 41d1ae4f2bdfa358aca3fca2b91ea2b57e4c3405..4611062f1b09ff2dbad02f93b9cc7f9920c32f5e 100644 --- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.gradle +++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.gradle @@ -18,7 +18,7 @@ repositories { } } -def apacheBeamVersion = '2.22.0' //'2.27.0' // '2.34.0' +def apacheBeamVersion = '2.35.0' dependencies { // These dependencies are used internally, and not exposed to consumers on their own compile classpath. diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.samza.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.samza.gradle index 73e916ccc867b9b3316776192f0dab56fa0710f0..44c59317472686cae88d6992382ae081c9b64ace 100644 --- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.samza.gradle +++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.beam.samza.gradle @@ -3,7 +3,8 @@ plugins { } dependencies { - implementation('org.apache.beam:beam-runners-samza:2.22.0') { + implementation('org.apache.beam:beam-runners-samza:2.35.0') { exclude group: 'org.apache.samza', module: 'samza-yarn_2.11' } + implementation 'org.apache.samza:samza-kafka_2.11:1.5.0' } \ No newline at end of file diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle index f5e93dd88d2234f8a9b0d6fea880f47d652dccfa..258d1a82d002184fe96a9df19b7d99806da50d28 100644 --- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle +++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.flink.gradle @@ -20,7 +20,7 @@ shadowJar { tasks.distZip.enabled = false ext { - flinkVersion = '1.12.2' + flinkVersion = '1.13.5' scalaBinaryVersion = '2.12' } diff --git a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle index da2d42176ac0ddc9a157f843e3268b37ac4397e2..112ac662798d5a1e41f146014dd95bdaaba3a264 100644 --- a/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle +++ b/theodolite-benchmarks/buildSrc/src/main/groovy/theodolite.kstreams.gradle @@ -22,7 +22,7 @@ dependencies { // These dependencies are used internally, and not exposed to consumers on their own compile classpath. implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true } implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true } - implementation 'org.apache.kafka:kafka-streams:2.6.0' // enable TransformerSuppliers + implementation 'org.apache.kafka:kafka-streams:3.1.0' implementation 'com.google.code.gson:gson:2.8.2' implementation 'com.google.guava:guava:24.1-jre' implementation 'org.slf4j:slf4j-simple:1.7.25' diff --git a/theodolite-benchmarks/config/spotbugs-exclude-filter.xml b/theodolite-benchmarks/config/spotbugs-exclude-filter.xml index 38cd4b6914b3f0cbdeac2a22bf29ce0d7cf487ee..4bc9e048fb20e27feb793d4c6c398345cd599cff 100644 --- a/theodolite-benchmarks/config/spotbugs-exclude-filter.xml +++ b/theodolite-benchmarks/config/spotbugs-exclude-filter.xml @@ -5,4 +5,7 @@ <Rank value="16" /> </Match> + <!-- Temporary disabled due to potential false positive reported in https://github.com/spotbugs/spotbugs/issues/1923. --> + <Bug code="NP" /> + </FindBugsFilter> \ No newline at end of file diff --git a/theodolite-benchmarks/docker-test/README.md b/theodolite-benchmarks/docker-test/README.md index fd1e9bf4730f897273be45a022ad2adeae1b7e6e..5d7ca3f4ac470202579f154fe8f066a246c84d23 100644 --- a/theodolite-benchmarks/docker-test/README.md +++ b/theodolite-benchmarks/docker-test/README.md @@ -36,3 +36,19 @@ the host, for example, from the IDE or Gradle. In such cases, the following adju You can now connect to Kafka from your host system with bootstrap server `localhost:19092` and contact the Schema Registry via `localhost:8081`. **Pay attention to the Kafka port, which is *19092* instead of the default one *9092*.** + +## Running Smoke Tests + +The `smoketest-runner.sh` script can be used to run a simple test for a specific Docker Compose file. You can call it with + +```sh +./smoketest-runner.sh <docker-compose-dir> +``` + +where `<docker-compose-dir>` is the directory of a Docker-Compose file, for example, `uc2-beam-samza`. The script exists with a zero exit code in case of success and a non-zero exit code otherwise. + +You can also run the set of all smoke test with: + +```sh +./smoketest-runner-all.sh +``` diff --git a/theodolite-benchmarks/docker-test/smoketest-runner-all.sh b/theodolite-benchmarks/docker-test/smoketest-runner-all.sh new file mode 100755 index 0000000000000000000000000000000000000000..0129a485d98a90d453b284408b755986f64208de --- /dev/null +++ b/theodolite-benchmarks/docker-test/smoketest-runner-all.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +find . -name 'test.sh' -type f -exec dirname {} \; | + sort | + xargs -I %s sh -c "./smoketest-runner.sh %s 1>&2; echo $?" | + sort | + awk 'BEGIN {count[0]=0; count[1]=0} {count[$1!=0]++} END {print count[0] " tests successful, " count[1] " test failed."; exit count[1]}' diff --git a/theodolite-benchmarks/docker-test/smoketest-runner.sh b/theodolite-benchmarks/docker-test/smoketest-runner.sh new file mode 100755 index 0000000000000000000000000000000000000000..3167c90a567eed3cc2678c80c722dbd0b8684f1e --- /dev/null +++ b/theodolite-benchmarks/docker-test/smoketest-runner.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +COMPOSE_FILE_PATH=$1 +echo "Run test for '$COMPOSE_FILE_PATH'." + +cd $COMPOSE_FILE_PATH +docker-compose pull -q +docker-compose up -d kafka zookeeper schema-registry +sleep 30s +docker-compose up -d +sleep 5s +docker-compose ps + +if test -f "./test.sh"; then + #timeout --foreground 3m ./test.sh + ./test.sh + RETURN=$? +else + RETURN=$? + echo "test.sh does not exists for '$COMPOSE_FILE_PATH'." +fi +if [ $RETURN -eq 0 ]; then + echo "Test for '$COMPOSE_FILE_PATH' has passed." +else + echo "Test for '$COMPOSE_FILE_PATH' has failed." +fi + +docker-compose down +exit $RETURN diff --git a/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml index 1b683b4ca65a2582aa6a4d68444c4bbef7895b73..69de6049aca50a6a8e10a9e709b38d322bc8cf61 100644 --- a/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-beam-flink/docker-compose.yml @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc1-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc1-beam-flink/test.sh b/theodolite-benchmarks/docker-test/uc1-beam-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..ebbecd1c5336c5dd907db11b8c8c45924e5924a8 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-beam-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +sleep 55s # to let the benchmark and produce some output +docker-compose logs --tail 100 benchmark-taskmanager | + sed -n "s/^.*Key:\s\(\S*\), Value:\s\(\S*\).*$/\2/p" | + tee /dev/stderr | + jq .identifier | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml index f5213799daa2d51eea53e794becdffc151a4da56..2212c3b539045114f31760d605ad928e237ed924 100644 --- a/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-beam-samza/docker-compose.yml @@ -22,18 +22,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc1-beam-samza:${THEODOLITE_TAG:-latest} scale: 1 diff --git a/theodolite-benchmarks/docker-test/uc1-beam-samza/test.sh b/theodolite-benchmarks/docker-test/uc1-beam-samza/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..ed17db3a44d5c4a8dacfbc956c2f36dd47503508 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-beam-samza/test.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +sleep 55s # to let the benchmark and produce some output +docker-compose logs --tail 100 benchmark | + sed -n "s/^.*Key:\s\(\S*\), Value:\s\(\S*\).*$/\2/p" | + tee /dev/stderr | + jq .identifier | + sort | + uniq | + wc -l | + grep "\b10\b" + + diff --git a/theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-flink/docker-compose.yml similarity index 90% rename from theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc1-flink/docker-compose.yml index 6c661bb49cb4173357acc89c06783fe5e0a2ce49..55e359665de56cb03a5049c299761cb07690df30 100755 --- a/theodolite-benchmarks/docker-test/uc1-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-flink/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc1-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc1-flink/test.sh b/theodolite-benchmarks/docker-test/uc1-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..7c7f11a94f42d56d91d383f27d58ad9a09a918e5 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +sleep 55s # to let the benchmark and produce some output +docker-compose logs --tail 100 benchmark-taskmanager | + sed -n "s/^.*Record:\s\(\S*\)$/\1/p" | + tee /dev/stderr | + jq .identifier | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc1-kstreams/docker-compose.yml similarity index 83% rename from theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc1-kstreams/docker-compose.yml index 25c19e35e5dae1807ef46fb8ade4e888dff0c2d8..c85ce305c2f1383a77d4c405d52089ec1d2b02a6 100755 --- a/theodolite-benchmarks/docker-test/uc1-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc1-kstreams/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,20 +20,25 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: - image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:${THEODOLITE_TAG:-latest} + image: ghcr.io/cau-se/theodolite-uc1-kstreams-app:latest depends_on: - schema-registry - kafka diff --git a/theodolite-benchmarks/docker-test/uc1-kstreams/test.sh b/theodolite-benchmarks/docker-test/uc1-kstreams/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..da711e3fac1b1d664b1c8487687ceacdddce6efa --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc1-kstreams/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +sleep 55s # to let the benchmark and produce some output +docker-compose logs --tail 100 benchmark | + sed -n "s/^.*Record:\s\(\S*\)$/\1/p" | + tee /dev/stderr | + jq .identifier | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml index 8427161e43faa920e011973d76f32f9cc9d62f8c..79f6f040144d3c4da56d469095ad3ddce90026a2 100644 --- a/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-beam-flink/docker-compose.yml @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc2-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc2-beam-flink/test.sh b/theodolite-benchmarks/docker-test/uc2-beam-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..3c33834b0b21bc5dbe4e6a7c3ff947121bb2ce71 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-beam-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml index 820bb25dd370cf6c7410b20fbdbdb1d4281f47d3..cc6bc7a7112c35f11ce9cfd27d09aebe401c8c51 100644 --- a/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-beam-samza/docker-compose.yml @@ -22,18 +22,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc2-beam-samza:${THEODOLITE_TAG:-latest} scale: 1 diff --git a/theodolite-benchmarks/docker-test/uc2-beam-samza/test.sh b/theodolite-benchmarks/docker-test/uc2-beam-samza/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..3c33834b0b21bc5dbe4e6a7c3ff947121bb2ce71 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-beam-samza/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-flink/docker-compose.yml similarity index 89% rename from theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc2-flink/docker-compose.yml index e51c544bfd64c6f540f5a84a5f617de745a1a6b6..1397c8ab0e7bb8d7ee67b7fa9c18143c404fd9ff 100755 --- a/theodolite-benchmarks/docker-test/uc2-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-flink/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka - #ports: - # - "8081:8081" + restart: "on-failure" expose: - "8081" + #ports: + # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc2-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc2-flink/test.sh b/theodolite-benchmarks/docker-test/uc2-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..3c33834b0b21bc5dbe4e6a7c3ff947121bb2ce71 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc2-kstreams/docker-compose.yml similarity index 86% rename from theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc2-kstreams/docker-compose.yml index 4cc4978d844a8b361af4061ab41d1adc4e7a1813..efdba90bef634bab76012316f67b0f9be9f79c77 100755 --- a/theodolite-benchmarks/docker-test/uc2-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc2-kstreams/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka - #ports: - # - "8081:8081" + restart: "on-failure" expose: - "8081" + #ports: + # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc2-kstreams-app:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc2-kstreams/test.sh b/theodolite-benchmarks/docker-test/uc2-kstreams/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..3c33834b0b21bc5dbe4e6a7c3ff947121bb2ce71 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc2-kstreams/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 20 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml index b9e934b61f9742ad4e83601ff13707be0641c875..e68dc94d4185d789272279e8fa2e5d178ffdd14b 100644 --- a/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-beam-flink/docker-compose.yml @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc3-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc3-beam-flink/test.sh b/theodolite-benchmarks/docker-test/uc3-beam-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..0612838a3973a302c9acb3bbfa6b8d59ea2596c5 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc3-beam-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 600 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml index 3ae3a2faa7c0849b76ea838b09ba0254bdfea936..bf120f31dbfda2384b314ba4a90a25362f37b6c2 100644 --- a/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-beam-samza/docker-compose.yml @@ -22,18 +22,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc3-beam-samza:${THEODOLITE_TAG:-latest} scale: 1 diff --git a/theodolite-benchmarks/docker-test/uc3-beam-samza/test.sh b/theodolite-benchmarks/docker-test/uc3-beam-samza/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..0612838a3973a302c9acb3bbfa6b8d59ea2596c5 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc3-beam-samza/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 600 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-flink/docker-compose.yml similarity index 89% rename from theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc3-flink/docker-compose.yml index 42c55950fbea022f00019699ae72678fbe88b8cb..0c83d8a25d19d4989f982d251893635ea1222cb0 100755 --- a/theodolite-benchmarks/docker-test/uc3-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-flink/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka - #ports: - # - "8081:8081" + restart: "on-failure" expose: - "8081" + #ports: + # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc3-workload-generator:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc3-kstreams/docker-compose.yml similarity index 85% rename from theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc3-kstreams/docker-compose.yml index f943de372b75bf64df0028679d56950fedfaec48..7da1fa3b420e2dfa0d6d357723583bc3a256823e 100755 --- a/theodolite-benchmarks/docker-test/uc3-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc3-kstreams/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka - #ports: - # - "8081:8081" + restart: "on-failure" expose: - "8081" + #ports: + # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc3-kstreams-app:${THEODOLITE_TAG:-latest} depends_on: diff --git a/theodolite-benchmarks/docker-test/uc3-kstreams/test.sh b/theodolite-benchmarks/docker-test/uc3-kstreams/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..0612838a3973a302c9acb3bbfa6b8d59ea2596c5 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc3-kstreams/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=s -r http://schema-registry:8081 -f '%k:%s\n' -c 600 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b10\b" diff --git a/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml index b0d18dfe34c23847b3cc0b04d788fdc4b3552e68..2968b718eb06ade4d42bbe104e214d573881ac14 100644 --- a/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-beam-flink/docker-compose.yml @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc4-workload-generator:${THEODOLITE_TAG:-latest} depends_on: @@ -43,7 +48,7 @@ services: KAFKA_BOOTSTRAP_SERVERS: kafka:9092 SCHEMA_REGISTRY_URL: http://schema-registry:8081 NUM_SENSORS: 4 - NUM_NESTED_GROUPS: 4 + NUM_NESTED_GROUPS: 3 benchmark-jobmanager: image: ghcr.io/cau-se/theodolite-uc4-beam-flink:${THEODOLITE_TAG:-latest} #ports: diff --git a/theodolite-benchmarks/docker-test/uc4-beam-flink/test.sh b/theodolite-benchmarks/docker-test/uc4-beam-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..6a4c6dbf4f583e7598baefae8f48136bb2113630 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-beam-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 2000 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b21\b" diff --git a/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml index b9e4142cb0be2701f7772cff5725b524aaffcbc7..d236af7d284ebb085c78110feb6001cb28d18290 100644 --- a/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-beam-samza/docker-compose.yml @@ -22,18 +22,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc4-beam-samza:${THEODOLITE_TAG:-latest} scale: 1 @@ -57,4 +62,4 @@ services: KAFKA_BOOTSTRAP_SERVERS: kafka:9092 SCHEMA_REGISTRY_URL: http://schema-registry:8081 NUM_SENSORS: 4 - NUM_NESTED_GROUPS: 4 + NUM_NESTED_GROUPS: 3 diff --git a/theodolite-benchmarks/docker-test/uc4-beam-samza/test.sh b/theodolite-benchmarks/docker-test/uc4-beam-samza/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..6a4c6dbf4f583e7598baefae8f48136bb2113630 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-beam-samza/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 2000 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b21\b" diff --git a/theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-flink/docker-compose.yml similarity index 89% rename from theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc4-flink/docker-compose.yml index 5a5a5924ef13cc9691b267ce6169bdb85a1e267b..24e5acdf94d1ba3d5c2807d172ba054309a2ef4a 100755 --- a/theodolite-benchmarks/docker-test/uc4-flink-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-flink/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 load-generator: image: ghcr.io/cau-se/theodolite-uc4-workload-generator:${THEODOLITE_TAG:-latest} depends_on: @@ -43,7 +48,7 @@ services: KAFKA_BOOTSTRAP_SERVERS: kafka:9092 SCHEMA_REGISTRY_URL: http://schema-registry:8081 NUM_SENSORS: 4 - NUM_NESTED_GROUPS: 4 + NUM_NESTED_GROUPS: 3 benchmark-jobmanager: image: ghcr.io/cau-se/theodolite-uc4-flink:${THEODOLITE_TAG:-latest} #ports: diff --git a/theodolite-benchmarks/docker-test/uc4-flink/test.sh b/theodolite-benchmarks/docker-test/uc4-flink/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..0a478cf6641a1333f65281ae43cb525e32cb2510 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-flink/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 500 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b21\b" diff --git a/theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml b/theodolite-benchmarks/docker-test/uc4-kstreams/docker-compose.yml similarity index 86% rename from theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml rename to theodolite-benchmarks/docker-test/uc4-kstreams/docker-compose.yml index 1818505787f662d57f1a96c8c79b03d61cf9da2a..6aaa02990841547edb6059e4e2fbf3b28b50985c 100755 --- a/theodolite-benchmarks/docker-test/uc4-kstreams-docker-compose/docker-compose.yml +++ b/theodolite-benchmarks/docker-test/uc4-kstreams/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zookeeper: image: confluentinc/cp-zookeeper @@ -20,18 +20,23 @@ services: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: 30000 KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" KAFKA_CREATE_TOPICS: "input:3:1,output:3:1,configuration:3:1,aggregation-feedback:3:1" + kcat: + image: edenhill/kcat:1.7.1 + entrypoint: "sh" + tty: true schema-registry: - image: confluentinc/cp-schema-registry:5.3.1 + image: confluentinc/cp-schema-registry:7.0.1 depends_on: - zookeeper - kafka + restart: "on-failure" expose: - "8081" #ports: # - 8081:8081 environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181' + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 benchmark: image: ghcr.io/cau-se/theodolite-uc4-kstreams-app:${THEODOLITE_TAG:-latest} depends_on: @@ -51,4 +56,4 @@ services: KAFKA_BOOTSTRAP_SERVERS: kafka:9092 SCHEMA_REGISTRY_URL: http://schema-registry:8081 NUM_SENSORS: 4 - NUM_NESTED_GROUPS: 4 \ No newline at end of file + NUM_NESTED_GROUPS: 3 \ No newline at end of file diff --git a/theodolite-benchmarks/docker-test/uc4-kstreams/test.sh b/theodolite-benchmarks/docker-test/uc4-kstreams/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..9b9dee7dc78e7a587b9f2e5b778066e5bc099755 --- /dev/null +++ b/theodolite-benchmarks/docker-test/uc4-kstreams/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +until docker-compose exec -T kcat kcat -L -b kafka:9092 -t output -J | jq -r '.topics[0].partitions | length' | grep "\b3\b"; do sleep 5s; done + +docker-compose exec -T kcat kcat -C -b kafka:9092 -t output -s key=s -s value=avro -r http://schema-registry:8081 -f '%k:%s\n' -c 32 | + tee /dev/stderr | + awk -F ':' '!/^%/ {print $1}' | + sort | + uniq | + wc -l | + grep "\b16\b" diff --git a/theodolite-benchmarks/flink-commons/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/flink-commons/.settings/org.eclipse.jdt.ui.prefs index 66b402b58f39b79066638ce679c27c0378d5be54..174249a98f9d91ce2cbf2bb64b27c09b37f05d9f 100644 --- a/theodolite-benchmarks/flink-commons/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/flink-commons/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/flink-commons/build.gradle b/theodolite-benchmarks/flink-commons/build.gradle index a3a4a35752006bb10e15ff508ce0b37f70adc57d..bd07f1ca51ae6b781a92ae06ba8b2555fa83b873 100644 --- a/theodolite-benchmarks/flink-commons/build.gradle +++ b/theodolite-benchmarks/flink-commons/build.gradle @@ -3,7 +3,7 @@ plugins { } ext { - flinkVersion = '1.12.0' + flinkVersion = '1.13.5' scalaBinaryVersion = '2.12' } diff --git a/theodolite-benchmarks/http-bridge/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/http-bridge/.settings/org.eclipse.jdt.ui.prefs new file mode 100644 index 0000000000000000000000000000000000000000..a375cb792eeb842ecfd1f789fbf6a716df43e9c8 --- /dev/null +++ b/theodolite-benchmarks/http-bridge/.settings/org.eclipse.jdt.ui.prefs @@ -0,0 +1,127 @@ +cleanup.add_default_serial_version_id=true +cleanup.add_generated_serial_version_id=false +cleanup.add_missing_annotations=true +cleanup.add_missing_deprecated_annotations=true +cleanup.add_missing_methods=false +cleanup.add_missing_nls_tags=false +cleanup.add_missing_override_annotations=true +cleanup.add_missing_override_annotations_interface_methods=true +cleanup.add_serial_version_id=false +cleanup.always_use_blocks=true +cleanup.always_use_parentheses_in_expressions=false +cleanup.always_use_this_for_non_static_field_access=true +cleanup.always_use_this_for_non_static_method_access=true +cleanup.convert_functional_interfaces=false +cleanup.convert_to_enhanced_for_loop=true +cleanup.correct_indentation=true +cleanup.format_source_code=true +cleanup.format_source_code_changes_only=false +cleanup.insert_inferred_type_arguments=false +cleanup.make_local_variable_final=true +cleanup.make_parameters_final=true +cleanup.make_private_fields_final=true +cleanup.make_type_abstract_if_missing_method=false +cleanup.make_variable_declarations_final=true +cleanup.never_use_blocks=false +cleanup.never_use_parentheses_in_expressions=true +cleanup.organize_imports=true +cleanup.qualify_static_field_accesses_with_declaring_class=false +cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +cleanup.qualify_static_member_accesses_with_declaring_class=true +cleanup.qualify_static_method_accesses_with_declaring_class=false +cleanup.remove_private_constructors=true +cleanup.remove_redundant_modifiers=false +cleanup.remove_redundant_semicolons=true +cleanup.remove_redundant_type_arguments=true +cleanup.remove_trailing_whitespaces=true +cleanup.remove_trailing_whitespaces_all=true +cleanup.remove_trailing_whitespaces_ignore_empty=false +cleanup.remove_unnecessary_casts=true +cleanup.remove_unnecessary_nls_tags=true +cleanup.remove_unused_imports=true +cleanup.remove_unused_local_variables=false +cleanup.remove_unused_private_fields=true +cleanup.remove_unused_private_members=false +cleanup.remove_unused_private_methods=true +cleanup.remove_unused_private_types=true +cleanup.sort_members=false +cleanup.sort_members_all=false +cleanup.use_anonymous_class_creation=false +cleanup.use_blocks=true +cleanup.use_blocks_only_for_return_and_throw=false +cleanup.use_lambda=true +cleanup.use_parentheses_in_expressions=true +cleanup.use_this_for_non_static_field_access=true +cleanup.use_this_for_non_static_field_access_only_if_necessary=false +cleanup.use_this_for_non_static_method_access=true +cleanup.use_this_for_non_static_method_access_only_if_necessary=false +cleanup_profile=_CAU-SE-Style +cleanup_settings_version=2 +eclipse.preferences.version=1 +editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true +formatter_profile=_CAU-SE-Style +formatter_settings_version=21 +org.eclipse.jdt.ui.ignorelowercasenames=true +org.eclipse.jdt.ui.importorder=; +org.eclipse.jdt.ui.ondemandthreshold=99 +org.eclipse.jdt.ui.staticondemandthreshold=99 +sp_cleanup.add_default_serial_version_id=true +sp_cleanup.add_generated_serial_version_id=false +sp_cleanup.add_missing_annotations=true +sp_cleanup.add_missing_deprecated_annotations=true +sp_cleanup.add_missing_methods=false +sp_cleanup.add_missing_nls_tags=false +sp_cleanup.add_missing_override_annotations=true +sp_cleanup.add_missing_override_annotations_interface_methods=true +sp_cleanup.add_serial_version_id=false +sp_cleanup.always_use_blocks=true +sp_cleanup.always_use_parentheses_in_expressions=false +sp_cleanup.always_use_this_for_non_static_field_access=true +sp_cleanup.always_use_this_for_non_static_method_access=true +sp_cleanup.convert_functional_interfaces=false +sp_cleanup.convert_to_enhanced_for_loop=true +sp_cleanup.correct_indentation=true +sp_cleanup.format_source_code=true +sp_cleanup.format_source_code_changes_only=false +sp_cleanup.insert_inferred_type_arguments=false +sp_cleanup.make_local_variable_final=true +sp_cleanup.make_parameters_final=true +sp_cleanup.make_private_fields_final=true +sp_cleanup.make_type_abstract_if_missing_method=false +sp_cleanup.make_variable_declarations_final=true +sp_cleanup.never_use_blocks=false +sp_cleanup.never_use_parentheses_in_expressions=true +sp_cleanup.on_save_use_additional_actions=true +sp_cleanup.organize_imports=true +sp_cleanup.qualify_static_field_accesses_with_declaring_class=false +sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true +sp_cleanup.qualify_static_member_accesses_with_declaring_class=true +sp_cleanup.qualify_static_method_accesses_with_declaring_class=false +sp_cleanup.remove_private_constructors=true +sp_cleanup.remove_redundant_modifiers=false +sp_cleanup.remove_redundant_semicolons=true +sp_cleanup.remove_redundant_type_arguments=true +sp_cleanup.remove_trailing_whitespaces=true +sp_cleanup.remove_trailing_whitespaces_all=true +sp_cleanup.remove_trailing_whitespaces_ignore_empty=false +sp_cleanup.remove_unnecessary_casts=true +sp_cleanup.remove_unnecessary_nls_tags=true +sp_cleanup.remove_unused_imports=true +sp_cleanup.remove_unused_local_variables=false +sp_cleanup.remove_unused_private_fields=true +sp_cleanup.remove_unused_private_members=false +sp_cleanup.remove_unused_private_methods=true +sp_cleanup.remove_unused_private_types=true +sp_cleanup.sort_members=false +sp_cleanup.sort_members_all=false +sp_cleanup.use_anonymous_class_creation=false +sp_cleanup.use_blocks=true +sp_cleanup.use_blocks_only_for_return_and_throw=false +sp_cleanup.use_lambda=true +sp_cleanup.use_parentheses_in_expressions=true +sp_cleanup.use_this_for_non_static_field_access=true +sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=false +sp_cleanup.use_this_for_non_static_method_access=true +sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=false diff --git a/theodolite-benchmarks/http-bridge/.settings/qa.eclipse.plugin.checkstyle.prefs b/theodolite-benchmarks/http-bridge/.settings/qa.eclipse.plugin.checkstyle.prefs new file mode 100644 index 0000000000000000000000000000000000000000..87860c815222845c1d264d7d0ce498d3397f8280 --- /dev/null +++ b/theodolite-benchmarks/http-bridge/.settings/qa.eclipse.plugin.checkstyle.prefs @@ -0,0 +1,4 @@ +configFilePath=../config/checkstyle.xml +customModulesJarPaths= +eclipse.preferences.version=1 +enabled=true diff --git a/theodolite-benchmarks/http-bridge/.settings/qa.eclipse.plugin.pmd.prefs b/theodolite-benchmarks/http-bridge/.settings/qa.eclipse.plugin.pmd.prefs new file mode 100644 index 0000000000000000000000000000000000000000..efbcb8c9e5d449194a48ca1ea42b7d807b573db9 --- /dev/null +++ b/theodolite-benchmarks/http-bridge/.settings/qa.eclipse.plugin.pmd.prefs @@ -0,0 +1,4 @@ +customRulesJars= +eclipse.preferences.version=1 +enabled=true +ruleSetFilePath=../config/pmd.xml diff --git a/theodolite-benchmarks/http-bridge/Dockerfile b/theodolite-benchmarks/http-bridge/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..b31dbcdef48f3b9eadf81a35c95e441c4b54955b --- /dev/null +++ b/theodolite-benchmarks/http-bridge/Dockerfile @@ -0,0 +1,6 @@ +FROM openjdk:11-slim + +ADD build/distributions/http-bridge.tar / + +CMD JAVA_OPTS="$JAVA_OPTS -Dorg.slf4j.simpleLogger.defaultLogLevel=$LOG_LEVEL" \ + /http-bridge/bin/http-bridge \ No newline at end of file diff --git a/theodolite-benchmarks/http-bridge/build.gradle b/theodolite-benchmarks/http-bridge/build.gradle new file mode 100644 index 0000000000000000000000000000000000000000..0377eefc76b456d8e0f94087b06d0c2689f977cf --- /dev/null +++ b/theodolite-benchmarks/http-bridge/build.gradle @@ -0,0 +1,31 @@ +plugins { + // common java conventions + id 'theodolite.java-conventions' + + // make executable + id 'application' +} + +tasks.distZip.enabled = false + +repositories { + mavenCentral() + maven { + url "https://oss.sonatype.org/content/repositories/snapshots/" + } + maven { + url 'https://packages.confluent.io/maven/' + } +} + +dependencies { + implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true } + implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true } + implementation project(':load-generator-commons') + + implementation 'io.javalin:javalin:4.3.0' + implementation 'com.google.code.gson:gson:2.8.2' + runtimeOnly 'org.slf4j:slf4j-simple:1.7.25' + + testImplementation 'junit:junit:4.12' +} diff --git a/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/Deserializer.java b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/Deserializer.java new file mode 100644 index 0000000000000000000000000000000000000000..f25c120d2165c4a1f747fdba32de43d4e4d157a6 --- /dev/null +++ b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/Deserializer.java @@ -0,0 +1,13 @@ +package theodolite.commons.httpbridge; + +/** + * A class for converting objects to strings. + * + * @param <T> Type to be deserialized from. + */ +@FunctionalInterface +public interface Deserializer<T> { + + T deserialize(String json); + +} diff --git a/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/Endpoint.java b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/Endpoint.java new file mode 100644 index 0000000000000000000000000000000000000000..43850d80699a0db0b0fcebd76f625a17f8133f30 --- /dev/null +++ b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/Endpoint.java @@ -0,0 +1,52 @@ +package theodolite.commons.httpbridge; + +import theodolite.commons.workloadgeneration.RecordSender; + +/** + * Class describing an endpoint of the {@link HttpBridge}, which converts JSON objects to Java + * objects and sends them using a {@link RecordSender}. + * + * @param <T> Type of objects this endpoint receives and converts. + */ +public class Endpoint<T> { + + private final String path; + + private final Deserializer<? extends T> recordDeserializer; + + private final RecordSender<? super T> recordSender; + + /** + * Create a new {@link Endpoint} at the given path. + */ + public Endpoint( + final String path, + final Deserializer<? extends T> recordDeserializer, + final RecordSender<? super T> recordSender) { + this.path = path; + this.recordDeserializer = recordDeserializer; + this.recordSender = recordSender; + } + + /** + * Create a new {@link Endpoint} at the given path with a {@link GsonDeserializer}. + */ + public Endpoint( + final String path, + final Class<T> recordType, + final RecordSender<? super T> recordSender) { + this.path = path; + this.recordDeserializer = new GsonDeserializer<>(recordType); + this.recordSender = recordSender; + } + + public String getPath() { + return this.path; + } + + public void convert(final String json) { + final T record = this.recordDeserializer.deserialize(json); + this.recordSender.send(record); + } + +} diff --git a/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/EnvVarHttpBridgeFactory.java b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/EnvVarHttpBridgeFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..97956b8d47185c90efdc03393c03c8c44aea2335 --- /dev/null +++ b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/EnvVarHttpBridgeFactory.java @@ -0,0 +1,59 @@ +package theodolite.commons.httpbridge; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import theodolite.commons.workloadgeneration.ConfigurationKeys; +import theodolite.commons.workloadgeneration.TitanKafkaSenderFactory; +import titan.ccp.model.records.ActivePowerRecord; + +class EnvVarHttpBridgeFactory { + + private static final String PORT_KEY = "PORT"; + private static final int PORT_DEFAULT = 8080; + + private static final String HOST_KEY = "HOST"; + private static final String HOST_DEFAULT = "0.0.0.0"; // NOPMD + + private static final String KAFKA_BOOTSTRAP_SERVERS_DEFAULT = "localhost:9092"; // NOPMD + private static final String KAFKA_TOPIC_DEFAULT = "input"; + private static final String SCHEMA_REGISTRY_URL_DEFAULT = "http://localhost:8081"; + + public HttpBridge create() { + final Endpoint<?> converter = new Endpoint<>( + "/", + ActivePowerRecord.class, + TitanKafkaSenderFactory.forKafkaConfig( + this.getKafkaBootstrapServer(), + this.getKafkaTopic(), + this.getSchemaRegistryUrl())); + return new HttpBridge(this.getHost(), this.getPort(), List.of(converter)); + } + + private String getHost() { + return Objects.requireNonNullElse(System.getenv(HOST_KEY), HOST_DEFAULT); + } + + private int getPort() { + return Optional.ofNullable(System.getenv(PORT_KEY)).map(Integer::parseInt).orElse(PORT_DEFAULT); + } + + private String getKafkaBootstrapServer() { + return Objects.requireNonNullElse( + System.getenv(ConfigurationKeys.KAFKA_BOOTSTRAP_SERVERS), + KAFKA_BOOTSTRAP_SERVERS_DEFAULT); + } + + private String getKafkaTopic() { + return Objects.requireNonNullElse( + System.getenv(ConfigurationKeys.KAFKA_INPUT_TOPIC), + KAFKA_TOPIC_DEFAULT); + } + + private String getSchemaRegistryUrl() { + return Objects.requireNonNullElse( + System.getenv(ConfigurationKeys.SCHEMA_REGISTRY_URL), + SCHEMA_REGISTRY_URL_DEFAULT); + } + +} diff --git a/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/GsonDeserializer.java b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/GsonDeserializer.java new file mode 100644 index 0000000000000000000000000000000000000000..42220617546527157d5463d6b9ce9208abc66d58 --- /dev/null +++ b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/GsonDeserializer.java @@ -0,0 +1,29 @@ +package theodolite.commons.httpbridge; + +import com.google.gson.Gson; + +/** + * A {@link Deserializer} based on GSON. + * + * @param <T> Type to be serialized from. + */ +public class GsonDeserializer<T> implements Deserializer<T> { + + private final Gson gson; + private final Class<T> targetClass; + + public GsonDeserializer(final Class<T> targetClass) { + this(new Gson(), targetClass); + } + + public GsonDeserializer(final Gson gson, final Class<T> targetClass) { + this.gson = gson; + this.targetClass = targetClass; + } + + @Override + public T deserialize(final String json) { + return this.gson.fromJson(json, this.targetClass); + } + +} diff --git a/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/HttpBridge.java b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/HttpBridge.java new file mode 100644 index 0000000000000000000000000000000000000000..d36e191c8b0a591107de796f511aa853063dff73 --- /dev/null +++ b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/HttpBridge.java @@ -0,0 +1,40 @@ +package theodolite.commons.httpbridge; + +import java.util.List; +import theodolite.commons.workloadgeneration.RecordSender; + +/** + * Class that creates a webserver with potentially multiple {@link Endpoint}s, which receives JSON + * objects at these endpoints, converts them to Java objects and send them using + * {@link RecordSender}s. + */ +public class HttpBridge { + + private final JavalinWebServer webServer; + + public HttpBridge(final String host, final int port, final List<Endpoint<?>> converters) { + this.webServer = new JavalinWebServer(converters, host, port); + } + + public void start() { + this.webServer.start(); + } + + public void stop() { + this.webServer.stop(); + } + + public void runAsStandalone() { + Runtime.getRuntime().addShutdownHook(new Thread(() -> this.stop())); + this.start(); + } + + public static HttpBridge fromEnvironment() { + return new EnvVarHttpBridgeFactory().create(); + } + + public static void main(final String[] args) { + HttpBridge.fromEnvironment().runAsStandalone(); + } + +} diff --git a/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/JavalinWebServer.java b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/JavalinWebServer.java new file mode 100644 index 0000000000000000000000000000000000000000..c23a17588d661fc5d1c6e9eb294d2d29fc165675 --- /dev/null +++ b/theodolite-benchmarks/http-bridge/src/main/java/theodolite/commons/httpbridge/JavalinWebServer.java @@ -0,0 +1,53 @@ +package theodolite.commons.httpbridge; + +import io.javalin.Javalin; +import java.util.Collection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of a webserver based on the Javalin framework. + */ +public class JavalinWebServer { + + private static final Logger LOGGER = LoggerFactory.getLogger(JavalinWebServer.class); + + private static final int HTTP_SUCCESS = 200; + + private final Javalin app = Javalin.create(); + + private final String host; + private final int port; + + /** + * Create a new instance, running on the specified host and port with the configured endpoints. + */ + public JavalinWebServer( + final Collection<Endpoint<?>> converters, + final String host, + final int port) { + this.host = host; + this.port = port; + this.configureRoutes(converters); + } + + private void configureRoutes(final Collection<Endpoint<?>> endpoints) { + for (final Endpoint<?> endpoint : endpoints) { + this.app.post(endpoint.getPath(), ctx -> { + final String record = ctx.body(); + LOGGER.debug("Received record at '{}': {}", ctx.path(), record); + endpoint.convert(record); + ctx.status(HTTP_SUCCESS); + }); + } + } + + public void start() { + this.app.start(this.host, this.port); + } + + public void stop() { + this.app.close(); + } + +} diff --git a/theodolite-benchmarks/kstreams-commons/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/kstreams-commons/.settings/org.eclipse.jdt.ui.prefs index 98b5ca8064a352aacfe2aebd13fbd0a87735fc3e..713419c8d3d74d3bd7fd05c3e839367753fcdee0 100644 --- a/theodolite-benchmarks/kstreams-commons/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/kstreams-commons/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/kstreams-commons/build.gradle b/theodolite-benchmarks/kstreams-commons/build.gradle index 7683ffe39314ec375eda0ed4e139d618d44a7328..167a75327b251af20b1142fe42c82b3bbedfe62b 100644 --- a/theodolite-benchmarks/kstreams-commons/build.gradle +++ b/theodolite-benchmarks/kstreams-commons/build.gradle @@ -17,7 +17,7 @@ dependencies { // implementation 'org.slf4j:slf4j-simple:1.7.25' implementation('org.industrial-devops:titan-ccp-common:0.1.0-SNAPSHOT') { changing = true } implementation('org.industrial-devops:titan-ccp-common-kafka:0.1.0-SNAPSHOT') { changing = true } - implementation 'org.apache.kafka:kafka-streams:2.6.0' + implementation 'org.apache.kafka:kafka-streams:3.1.0' // Use JUnit test framework testImplementation 'junit:junit:4.12' diff --git a/theodolite-benchmarks/kstreams-commons/src/main/java/theodolite/commons/kafkastreams/KafkaStreamsBuilder.java b/theodolite-benchmarks/kstreams-commons/src/main/java/theodolite/commons/kafkastreams/KafkaStreamsBuilder.java index 89bd3147f0d3bb7a5fecc5d8c7d277bd294494ad..fe3cf484a81ee3561ad17b6b25d218cd011f2d5d 100644 --- a/theodolite-benchmarks/kstreams-commons/src/main/java/theodolite/commons/kafkastreams/KafkaStreamsBuilder.java +++ b/theodolite-benchmarks/kstreams-commons/src/main/java/theodolite/commons/kafkastreams/KafkaStreamsBuilder.java @@ -70,18 +70,15 @@ public abstract class KafkaStreamsBuilder { // optional configurations this.setOptionalProperty(propBuilder, StreamsConfig.ACCEPTABLE_RECOVERY_LAG_CONFIG, - this.config::getLong, - p -> p >= 0); + this.config::getLong, p -> p >= 0); this.setOptionalProperty(propBuilder, StreamsConfig.BUFFERED_RECORDS_PER_PARTITION_CONFIG, this.config::getInt, p -> p > 0); this.setOptionalProperty(propBuilder, StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, - this.config::getInt, - p -> p >= 0); + this.config::getInt, p -> p >= 0); this.setOptionalProperty(propBuilder, StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, this.config::getInt, p -> p >= 0); this.setOptionalProperty(propBuilder, StreamsConfig.MAX_TASK_IDLE_MS_CONFIG, - this.config::getLong, - p -> p >= 0); + this.config::getLong, p -> p >= 0); this.setOptionalProperty(propBuilder, StreamsConfig.MAX_WARMUP_REPLICAS_CONFIG, this.config::getInt, p -> p >= 1); this.setOptionalProperty(propBuilder, StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, @@ -89,22 +86,28 @@ public abstract class KafkaStreamsBuilder { this.setOptionalProperty(propBuilder, StreamsConfig.NUM_STREAM_THREADS_CONFIG, this.config::getInt, p -> p > 0); this.setOptionalProperty(propBuilder, StreamsConfig.POLL_MS_CONFIG, - this.config::getLong, - p -> p >= 0); + this.config::getLong, p -> p >= 0); this.setOptionalProperty(propBuilder, StreamsConfig.PROCESSING_GUARANTEE_CONFIG, - this.config::getString, p -> StreamsConfig.AT_LEAST_ONCE.equals(p) - || StreamsConfig.EXACTLY_ONCE.equals(p) || StreamsConfig.EXACTLY_ONCE_BETA.equals(p)); + this.config::getString, this::validateProcessingGuarantee); this.setOptionalProperty(propBuilder, StreamsConfig.REPLICATION_FACTOR_CONFIG, this.config::getInt, p -> p >= 0); - if (this.config.containsKey(StreamsConfig.TOPOLOGY_OPTIMIZATION) - && this.config.getBoolean(StreamsConfig.TOPOLOGY_OPTIMIZATION)) { - propBuilder.set(StreamsConfig.TOPOLOGY_OPTIMIZATION, StreamsConfig.OPTIMIZE); + if (this.config.containsKey(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG) + && this.config.getBoolean(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG)) { + propBuilder.set(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.OPTIMIZE); } return propBuilder.build(); } + @SuppressWarnings("deprecation") + private boolean validateProcessingGuarantee(final String processingGuarantee) { + return StreamsConfig.AT_LEAST_ONCE.equals(processingGuarantee) + // We continue support EXACTLY_ONCE to allow benchmarking it against v2 + || StreamsConfig.EXACTLY_ONCE.equals(processingGuarantee) + || StreamsConfig.EXACTLY_ONCE_V2.equals(processingGuarantee); + } + /** * Method to implement a {@link Topology} for a {@code KafkaStreams} application. * @@ -116,7 +119,7 @@ public abstract class KafkaStreamsBuilder { * Builds the {@link KafkaStreams} instance. */ public KafkaStreams build() { - // Create the Kafka streams instance. + // Create the Kafka Streams instance. final Properties properties = this.buildProperties(); return new KafkaStreams(this.buildTopology(properties), properties); } diff --git a/theodolite-benchmarks/load-generator-commons/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/load-generator-commons/.settings/org.eclipse.jdt.ui.prefs index fa98ca63d77bdee891150bd6713f70197a75cefc..a375cb792eeb842ecfd1f789fbf6a716df43e9c8 100644 --- a/theodolite-benchmarks/load-generator-commons/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/load-generator-commons/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/load-generator-commons/src/test/java/theodolite/commons/workloadgeneration/HttpRecordSenderTest.java b/theodolite-benchmarks/load-generator-commons/src/test/java/theodolite/commons/workloadgeneration/HttpRecordSenderTest.java index 6d908d34b7c6b87254782b6ae8b0b8dc2a6d036e..0d331a900f5bd5c18dbeaf2fc2a249256151ce70 100644 --- a/theodolite-benchmarks/load-generator-commons/src/test/java/theodolite/commons/workloadgeneration/HttpRecordSenderTest.java +++ b/theodolite-benchmarks/load-generator-commons/src/test/java/theodolite/commons/workloadgeneration/HttpRecordSenderTest.java @@ -10,7 +10,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; import com.github.tomakehurst.wiremock.junit.WireMockRule; -import com.google.gson.Gson; import java.net.URI; import org.junit.Before; import org.junit.Rule; @@ -21,8 +20,6 @@ public class HttpRecordSenderTest { private HttpRecordSender<ActivePowerRecord> httpRecordSender; - private Gson gson; - @Rule public WireMockRule wireMockRule = new WireMockRule(options().dynamicPort()); @@ -30,7 +27,6 @@ public class HttpRecordSenderTest { public void setup() { this.httpRecordSender = new HttpRecordSender<>(URI.create("http://localhost:" + this.wireMockRule.port())); - this.gson = new Gson(); } @Test diff --git a/theodolite-benchmarks/settings.gradle b/theodolite-benchmarks/settings.gradle index ae4254e968a0bc09970752f95c6a40db86ae775c..2f6bd4b74cbf103b740d8d8f57e38ac9deafd3cf 100644 --- a/theodolite-benchmarks/settings.gradle +++ b/theodolite-benchmarks/settings.gradle @@ -34,3 +34,4 @@ include 'uc4-flink' include 'uc4-beam-flink' include 'uc4-beam-samza' +include 'http-bridge' diff --git a/theodolite-benchmarks/uc1-beam-samza/Dockerfile b/theodolite-benchmarks/uc1-beam-samza/Dockerfile index 9b729060532ea3a242ac3084ba0bebf88ca2e9b6..cf6ef6675464e3c9d37db492b39fd8a71ec60e63 100644 --- a/theodolite-benchmarks/uc1-beam-samza/Dockerfile +++ b/theodolite-benchmarks/uc1-beam-samza/Dockerfile @@ -5,5 +5,5 @@ ENV MAX_SOURCE_PARALLELISM=1024 ADD build/distributions/uc1-beam-samza.tar / ADD samza-standalone.properties / -CMD /uc1-beam-samza/bin/uc1-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}" +CMD /uc1-beam-samza/bin/uc1-beam-samza --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}" diff --git a/theodolite-benchmarks/uc1-beam/src/main/java/application/LogKeyValue.java b/theodolite-benchmarks/uc1-beam/src/main/java/application/LogKeyValue.java index 79566fd937b9c100663d426610b6ff476035ef87..251523441e339cbaf58c7e3a1b30e97cc354df18 100644 --- a/theodolite-benchmarks/uc1-beam/src/main/java/application/LogKeyValue.java +++ b/theodolite-benchmarks/uc1-beam/src/main/java/application/LogKeyValue.java @@ -8,7 +8,6 @@ import org.slf4j.LoggerFactory; /** * Logs all Key Value pairs. */ -@SuppressWarnings({"unused"}) public class LogKeyValue extends DoFn<KV<String, String>, KV<String, String>> { private static final long serialVersionUID = 4328743; private static final Logger LOGGER = LoggerFactory.getLogger(LogKeyValue.class); @@ -19,9 +18,7 @@ public class LogKeyValue extends DoFn<KV<String, String>, KV<String, String>> { @ProcessElement public void processElement(@Element final KV<String, String> kv, final OutputReceiver<KV<String, String>> out) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("Key: {}, Value: {}", kv.getKey(), kv.getValue()); - } + LOGGER.info("Key: {}, Value: {}", kv.getKey(), kv.getValue()); out.output(kv); } } diff --git a/theodolite-benchmarks/uc1-flink/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc1-flink/.settings/org.eclipse.jdt.ui.prefs index fa98ca63d77bdee891150bd6713f70197a75cefc..a375cb792eeb842ecfd1f789fbf6a716df43e9c8 100644 --- a/theodolite-benchmarks/uc1-flink/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc1-flink/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc1-flink/Dockerfile b/theodolite-benchmarks/uc1-flink/Dockerfile index 795b9e343a03cf0209e1625f5cbc3d45dcb77cda..fdaecd661e86275d670423351664e54221ce33cc 100644 --- a/theodolite-benchmarks/uc1-flink/Dockerfile +++ b/theodolite-benchmarks/uc1-flink/Dockerfile @@ -1,3 +1,3 @@ -FROM flink:1.12-scala_2.12-java11 +FROM flink:1.13-java11 ADD build/libs/uc1-flink-all.jar /opt/flink/usrlib/artifacts/uc1-flink-all.jar diff --git a/theodolite-benchmarks/uc1-kstreams/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc1-kstreams/.settings/org.eclipse.jdt.ui.prefs index fa98ca63d77bdee891150bd6713f70197a75cefc..a375cb792eeb842ecfd1f789fbf6a716df43e9c8 100644 --- a/theodolite-benchmarks/uc1-kstreams/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc1-kstreams/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc1-load-generator/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc1-load-generator/.settings/org.eclipse.jdt.ui.prefs index 4d01df75552c562406705858b6368ecf59d6e82f..ac23341bf71ac68df4183361493261758fd5dafb 100644 --- a/theodolite-benchmarks/uc1-load-generator/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc1-load-generator/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc2-beam-samza/Dockerfile b/theodolite-benchmarks/uc2-beam-samza/Dockerfile index 22855cea279819cacbf6eee253c30c60409fdba3..ae762791c40fc6981ce7e5fd08bea860ed9208ec 100644 --- a/theodolite-benchmarks/uc2-beam-samza/Dockerfile +++ b/theodolite-benchmarks/uc2-beam-samza/Dockerfile @@ -5,4 +5,4 @@ ENV MAX_SOURCE_PARALLELISM=1024 ADD build/distributions/uc2-beam-samza.tar / ADD samza-standalone.properties / -CMD /uc2-beam-samza/bin/uc2-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}" +CMD /uc2-beam-samza/bin/uc2-beam-samza --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}" diff --git a/theodolite-benchmarks/uc2-flink/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc2-flink/.settings/org.eclipse.jdt.ui.prefs index 4d01df75552c562406705858b6368ecf59d6e82f..ac23341bf71ac68df4183361493261758fd5dafb 100644 --- a/theodolite-benchmarks/uc2-flink/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc2-flink/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc2-flink/Dockerfile b/theodolite-benchmarks/uc2-flink/Dockerfile index 537ab28e2d4e5fb8edfc2760142acc33cc49b91d..01a85c57b00ea8bb4af8faa697708fd2b650de50 100644 --- a/theodolite-benchmarks/uc2-flink/Dockerfile +++ b/theodolite-benchmarks/uc2-flink/Dockerfile @@ -1,3 +1,3 @@ -FROM flink:1.12-scala_2.12-java11 +FROM flink:1.13-java11 ADD build/libs/uc2-flink-all.jar /opt/flink/usrlib/artifacts/uc2-flink-all.jar \ No newline at end of file diff --git a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java index d156d895d86bb01a31f96e08764df8b8df743c4d..7a97099c71a18449b7cc3f0413632b52fd5b69f5 100644 --- a/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java +++ b/theodolite-benchmarks/uc2-flink/src/main/java/theodolite/uc2/application/HistoryServiceFlinkJob.java @@ -5,7 +5,6 @@ import org.apache.commons.configuration2.Configuration; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.state.StateBackend; -import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; import org.apache.flink.streaming.api.windowing.time.Time; @@ -48,8 +47,6 @@ public final class HistoryServiceFlinkJob { } private void configureEnv() { - this.env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); - final boolean checkpointing = this.config.getBoolean(ConfigurationKeys.CHECKPOINTING, true); final int commitIntervalMs = this.config.getInt(ConfigurationKeys.COMMIT_INTERVAL_MS); if (checkpointing) { diff --git a/theodolite-benchmarks/uc2-kstreams/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc2-kstreams/.settings/org.eclipse.jdt.ui.prefs index 4d01df75552c562406705858b6368ecf59d6e82f..ac23341bf71ac68df4183361493261758fd5dafb 100644 --- a/theodolite-benchmarks/uc2-kstreams/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc2-kstreams/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc2-kstreams/src/main/java/theodolite/uc2/streamprocessing/TopologyBuilder.java b/theodolite-benchmarks/uc2-kstreams/src/main/java/theodolite/uc2/streamprocessing/TopologyBuilder.java index eda7c495a2cff6d58b62a8a6a74ea8e1b2d89aca..21dcf14a9322ce5a6381f96f22f5fadb85cc78f0 100644 --- a/theodolite-benchmarks/uc2-kstreams/src/main/java/theodolite/uc2/streamprocessing/TopologyBuilder.java +++ b/theodolite-benchmarks/uc2-kstreams/src/main/java/theodolite/uc2/streamprocessing/TopologyBuilder.java @@ -53,7 +53,7 @@ public class TopologyBuilder { Consumed.with(Serdes.String(), this.srAvroSerdeFactory.<ActivePowerRecord>forValues())) .groupByKey() - .windowedBy(TimeWindows.of(this.duration)) + .windowedBy(TimeWindows.ofSizeWithNoGrace(this.duration)) // .aggregate( // () -> 0.0, // (key, activePowerRecord, agg) -> agg + activePowerRecord.getValueInW(), diff --git a/theodolite-benchmarks/uc2-load-generator/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc2-load-generator/.settings/org.eclipse.jdt.ui.prefs index 4d01df75552c562406705858b6368ecf59d6e82f..ac23341bf71ac68df4183361493261758fd5dafb 100644 --- a/theodolite-benchmarks/uc2-load-generator/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc2-load-generator/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc3-beam-samza/Dockerfile b/theodolite-benchmarks/uc3-beam-samza/Dockerfile index d3e860bd72c54121d616bc5562d519e6e1e21dec..54979b8e1fa8aa9ac7d073302301bd10cbff5f34 100644 --- a/theodolite-benchmarks/uc3-beam-samza/Dockerfile +++ b/theodolite-benchmarks/uc3-beam-samza/Dockerfile @@ -5,4 +5,4 @@ ENV MAX_SOURCE_PARALLELISM=1024 ADD build/distributions/uc3-beam-samza.tar / ADD samza-standalone.properties / -CMD /uc3-beam-samza/bin/uc3-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}" +CMD /uc3-beam-samza/bin/uc3-beam-samza --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}" diff --git a/theodolite-benchmarks/uc3-flink/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc3-flink/.settings/org.eclipse.jdt.ui.prefs index 4d01df75552c562406705858b6368ecf59d6e82f..ac23341bf71ac68df4183361493261758fd5dafb 100644 --- a/theodolite-benchmarks/uc3-flink/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc3-flink/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc3-flink/Dockerfile b/theodolite-benchmarks/uc3-flink/Dockerfile index 34c6da692cb30b738adf47b9d4ca893e72f330e4..cef05c0296f55f0cf7391dd35dd1806ec0efa287 100644 --- a/theodolite-benchmarks/uc3-flink/Dockerfile +++ b/theodolite-benchmarks/uc3-flink/Dockerfile @@ -1,3 +1,3 @@ -FROM flink:1.12-scala_2.12-java11 +FROM flink:1.13-java11 ADD build/libs/uc3-flink-all.jar /opt/flink/usrlib/artifacts/uc3-flink-all.jar \ No newline at end of file diff --git a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java index 091b25674a2a31671ca68bd2076c694da9533d77..621146864fc84d032a7dc3c65fa253df9b940b2d 100644 --- a/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java +++ b/theodolite-benchmarks/uc3-flink/src/main/java/theodolite/uc3/application/HistoryServiceFlinkJob.java @@ -9,7 +9,6 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.state.StateBackend; -import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows; import org.apache.flink.streaming.api.windowing.time.Time; @@ -55,8 +54,6 @@ public final class HistoryServiceFlinkJob { } private void configureEnv() { - this.env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); - final boolean checkpointing = this.config.getBoolean(ConfigurationKeys.CHECKPOINTING, true); final int commitIntervalMs = this.config.getInt(ConfigurationKeys.COMMIT_INTERVAL_MS); if (checkpointing) { diff --git a/theodolite-benchmarks/uc3-kstreams/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc3-kstreams/.settings/org.eclipse.jdt.ui.prefs index fa98ca63d77bdee891150bd6713f70197a75cefc..a375cb792eeb842ecfd1f789fbf6a716df43e9c8 100644 --- a/theodolite-benchmarks/uc3-kstreams/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc3-kstreams/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc3-kstreams/src/main/java/theodolite/uc3/streamprocessing/TopologyBuilder.java b/theodolite-benchmarks/uc3-kstreams/src/main/java/theodolite/uc3/streamprocessing/TopologyBuilder.java index 1e976c07158720b3681d89413a5f277b1395f32d..4c63e21f3d9f1af6c9ef0363d7d01939faae9aef 100644 --- a/theodolite-benchmarks/uc3-kstreams/src/main/java/theodolite/uc3/streamprocessing/TopologyBuilder.java +++ b/theodolite-benchmarks/uc3-kstreams/src/main/java/theodolite/uc3/streamprocessing/TopologyBuilder.java @@ -60,17 +60,18 @@ public class TopologyBuilder { final Serde<HourOfDayKey> keySerde = HourOfDayKeySerde.create(); this.builder - .stream(this.inputTopic, - Consumed.with(Serdes.String(), - this.srAvroSerdeFactory.<ActivePowerRecord>forValues())) + .stream(this.inputTopic, Consumed.with( + Serdes.String(), + this.srAvroSerdeFactory.<ActivePowerRecord>forValues())) .selectKey((key, value) -> { final Instant instant = Instant.ofEpochMilli(value.getTimestamp()); final LocalDateTime dateTime = LocalDateTime.ofInstant(instant, this.zone); return keyFactory.createKey(value.getIdentifier(), dateTime); }) - .groupByKey( - Grouped.with(keySerde, this.srAvroSerdeFactory.forValues())) - .windowedBy(TimeWindows.of(this.aggregtionDuration).advanceBy(this.aggregationAdvance)) + .groupByKey(Grouped.with(keySerde, this.srAvroSerdeFactory.forValues())) + .windowedBy(TimeWindows + .ofSizeWithNoGrace(this.aggregtionDuration) + .advanceBy(this.aggregationAdvance)) .aggregate( () -> Stats.of(), (k, record, stats) -> StatsFactory.accumulate(stats, record.getValueInW()), diff --git a/theodolite-benchmarks/uc3-load-generator/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc3-load-generator/.settings/org.eclipse.jdt.ui.prefs index fa98ca63d77bdee891150bd6713f70197a75cefc..a375cb792eeb842ecfd1f789fbf6a716df43e9c8 100644 --- a/theodolite-benchmarks/uc3-load-generator/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc3-load-generator/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc4-beam-samza/Dockerfile b/theodolite-benchmarks/uc4-beam-samza/Dockerfile index ffb82ee09e99384d4914a0f86b6d9214fc161381..b59585f748b95cf62e59be01c8fa9dc0d919f43a 100644 --- a/theodolite-benchmarks/uc4-beam-samza/Dockerfile +++ b/theodolite-benchmarks/uc4-beam-samza/Dockerfile @@ -5,4 +5,4 @@ ENV MAX_SOURCE_PARALLELISM=1024 ADD build/distributions/uc4-beam-samza.tar / ADD samza-standalone.properties / -CMD /uc4-beam-samza/bin/uc4-beam-samza --configFactory=org.apache.samza.config.factories.PropertiesConfigFactory --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}" +CMD /uc4-beam-samza/bin/uc4-beam-samza --configFilePath=samza-standalone.properties --samzaExecutionEnvironment=STANDALONE --maxSourceParallelism=$MAX_SOURCE_PARALLELISM --enableMetrics=false --configOverride="{\"job.coordinator.zk.connect\":\"$SAMZA_JOB_COORDINATOR_ZK_CONNECT\"}" diff --git a/theodolite-benchmarks/uc4-beam/build.gradle b/theodolite-benchmarks/uc4-beam/build.gradle index 502e94fa737fb2ae1bab861407b27575cd8766ca..3e9d917cc3586e5df2c5645f1d2cbcf03e3993e4 100644 --- a/theodolite-benchmarks/uc4-beam/build.gradle +++ b/theodolite-benchmarks/uc4-beam/build.gradle @@ -2,4 +2,6 @@ plugins { id 'theodolite.beam' } - +dependencies { + implementation ('io.confluent:kafka-streams-avro-serde:5.3.2') +} diff --git a/theodolite-benchmarks/uc4-beam/src/main/java/application/DuplicateAsFlatMap.java b/theodolite-benchmarks/uc4-beam/src/main/java/application/DuplicateAsFlatMap.java index 7b66082c91b87c246d8c834249d2bc82545766f5..347d76dfb3d1d1f09f1091296a322a23bba67ec0 100644 --- a/theodolite-benchmarks/uc4-beam/src/main/java/application/DuplicateAsFlatMap.java +++ b/theodolite-benchmarks/uc4-beam/src/main/java/application/DuplicateAsFlatMap.java @@ -14,13 +14,18 @@ import titan.ccp.model.records.ActivePowerRecord; /** - * Duplicates the Kv containing the (Children,Parents) pair as a flat map. + * Duplicates the {@link KV} containing the (children,parents) pairs as flatMap. */ -public class DuplicateAsFlatMap extends DoFn - <KV<String, ActivePowerRecord>, KV<SensorParentKey, ActivePowerRecord>> { +public class DuplicateAsFlatMap + extends DoFn<KV<String, ActivePowerRecord>, KV<SensorParentKey, ActivePowerRecord>> { + private static final long serialVersionUID = -5132355515723961647L; - @StateId("parents") - private final StateSpec<ValueState<Set<String>>> parents = StateSpecs.value();//NOPMD + + private static final String STATE_STORE_NAME = "DuplicateParents"; + + @StateId(STATE_STORE_NAME) + private final StateSpec<ValueState<Set<String>>> parents = StateSpecs.value(); // NOPMD + private final PCollectionView<Map<String, Set<String>>> childParentPairMap; public DuplicateAsFlatMap(final PCollectionView<Map<String, Set<String>>> childParentPairMap) { @@ -28,21 +33,21 @@ public class DuplicateAsFlatMap extends DoFn this.childParentPairMap = childParentPairMap; } - /** - * Generate a KV-pair for every child-parent match. + * Generate a KV-pair for every child-parent match. */ @ProcessElement - public void processElement(@Element final KV<String, ActivePowerRecord> kv, - final OutputReceiver<KV<SensorParentKey, ActivePowerRecord>> out, - @StateId("parents") final ValueState<Set<String>> state, - final ProcessContext c) { + public void processElement( + @Element final KV<String, ActivePowerRecord> kv, + final OutputReceiver<KV<SensorParentKey, ActivePowerRecord>> out, + @StateId(STATE_STORE_NAME) final ValueState<Set<String>> state, + final ProcessContext c) { final ActivePowerRecord record = kv.getValue() == null ? null : kv.getValue(); final Set<String> newParents = - c.sideInput(childParentPairMap).get(kv.getKey()) == null + c.sideInput(this.childParentPairMap).get(kv.getKey()) == null ? Collections.emptySet() - : c.sideInput(childParentPairMap).get(kv.getKey()); + : c.sideInput(this.childParentPairMap).get(kv.getKey()); final Set<String> oldParents = MoreObjects.firstNonNull(state.read(), Collections.emptySet()); // Forward new Pairs if they exist diff --git a/theodolite-benchmarks/uc4-beam/src/main/java/application/Uc4BeamPipeline.java b/theodolite-benchmarks/uc4-beam/src/main/java/application/Uc4BeamPipeline.java index 7179fe5da937280d5baf72cd73cc392ef15a60e0..0c63e6f9322e5f70f1ad010de168f1a5292a45a4 100644 --- a/theodolite-benchmarks/uc4-beam/src/main/java/application/Uc4BeamPipeline.java +++ b/theodolite-benchmarks/uc4-beam/src/main/java/application/Uc4BeamPipeline.java @@ -66,8 +66,8 @@ public final class Uc4BeamPipeline extends AbstractPipeline { final Duration gracePeriod = Duration.standardSeconds(config.getInt(ConfigurationKeys.GRACE_PERIOD_MS)); - // Build kafka configuration - final Map<String, Object> consumerConfig = this.buildConsumerConfig(); + // Build Kafka configuration + final Map<String, Object> consumerConfig = super.buildConsumerConfig(); final Map<String, Object> configurationConfig = this.configurationConfig(config); // Set Coders for Classes that will be distributed @@ -77,25 +77,34 @@ public final class Uc4BeamPipeline extends AbstractPipeline { // Read from Kafka // ActivePowerRecords final KafkaActivePowerTimestampReader kafkaActivePowerRecordReader = - new KafkaActivePowerTimestampReader(this.bootstrapServer, this.inputTopic, consumerConfig); + new KafkaActivePowerTimestampReader( + this.bootstrapServer, + this.inputTopic, + consumerConfig); // Configuration Events final KafkaGenericReader<Event, String> kafkaConfigurationReader = new KafkaGenericReader<>( - this.bootstrapServer, configurationTopic, configurationConfig, - EventDeserializer.class, StringDeserializer.class); - - // Transform into AggregatedActivePowerRecords into ActivePowerRecords - final AggregatedToActive aggregatedToActive = new AggregatedToActive(); + this.bootstrapServer, + configurationTopic, + configurationConfig, + EventDeserializer.class, + StringDeserializer.class); // Write to Kafka final KafkaWriterTransformation<AggregatedActivePowerRecord> kafkaOutput = new KafkaWriterTransformation<>( - this.bootstrapServer, outputTopic, AggregatedActivePowerRecordSerializer.class); + this.bootstrapServer, + outputTopic, + AggregatedActivePowerRecordSerializer.class, + super.buildProducerConfig()); final KafkaWriterTransformation<AggregatedActivePowerRecord> kafkaFeedback = new KafkaWriterTransformation<>( - this.bootstrapServer, feedbackTopic, AggregatedActivePowerRecordSerializer.class); + this.bootstrapServer, + feedbackTopic, + AggregatedActivePowerRecordSerializer.class, + super.buildProducerConfig()); // Apply pipeline transformations final PCollection<KV<String, ActivePowerRecord>> values = this @@ -115,7 +124,10 @@ public final class Uc4BeamPipeline extends AbstractPipeline { .withBootstrapServers(this.bootstrapServer) .withTopic(feedbackTopic) .withKeyDeserializer(StringDeserializer.class) - .withValueDeserializer(AggregatedActivePowerRecordDeserializer.class) + .withValueDeserializerAndCoder( + AggregatedActivePowerRecordDeserializer.class, + AvroCoder.of(AggregatedActivePowerRecord.class)) + .withConsumerConfigUpdates(consumerConfig) .withTimestampPolicyFactory( (tp, previousWaterMark) -> new AggregatedActivePowerRecordEventTimePolicy( previousWaterMark)) @@ -123,11 +135,12 @@ public final class Uc4BeamPipeline extends AbstractPipeline { .apply("Apply Windows", Window.into(FixedWindows.of(duration))) // Convert into the correct data format .apply("Convert AggregatedActivePowerRecord to ActivePowerRecord", - MapElements.via(aggregatedToActive)) + MapElements.via(new AggregatedToActive())) .apply("Set trigger for feedback", Window .<KV<String, ActivePowerRecord>>configure() .triggering(Repeatedly.forever( - AfterProcessingTime.pastFirstElementInPane() + AfterProcessingTime + .pastFirstElementInPane() .plusDelayOf(triggerDelay))) .withAllowedLateness(gracePeriod) .discardingFiredPanes()); @@ -170,17 +183,13 @@ public final class Uc4BeamPipeline extends AbstractPipeline { .accumulatingFiredPanes()) .apply(View.asMap()); - final FilterNullValues filterNullValues = new FilterNullValues(); - // Build pairs of every sensor reading and parent final PCollection<KV<SensorParentKey, ActivePowerRecord>> flatMappedValues = inputCollection.apply( "Duplicate as flatMap", - ParDo.of(new DuplicateAsFlatMap(childParentPairMap)) - .withSideInputs(childParentPairMap)) + ParDo.of(new DuplicateAsFlatMap(childParentPairMap)).withSideInputs(childParentPairMap)) .apply("Filter only latest changes", Latest.perKey()) - .apply("Filter out null values", - Filter.by(filterNullValues)); + .apply("Filter out null values", Filter.by(new FilterNullValues())); final SetIdForAggregated setIdForAggregated = new SetIdForAggregated(); final SetKeyToGroup setKeyToGroup = new SetKeyToGroup(); @@ -204,8 +213,7 @@ public final class Uc4BeamPipeline extends AbstractPipeline { aggregations.apply("Write to aggregation results", kafkaOutput); - aggregations - .apply("Write to feedback topic", kafkaFeedback); + aggregations.apply("Write to feedback topic", kafkaFeedback); } @@ -217,14 +225,15 @@ public final class Uc4BeamPipeline extends AbstractPipeline { */ public Map<String, Object> configurationConfig(final Configuration config) { final Map<String, Object> consumerConfig = new HashMap<>(); - consumerConfig.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, + consumerConfig.put( + ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, config.getString(ConfigurationKeys.ENABLE_AUTO_COMMIT_CONFIG)); - consumerConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, - config - .getString(ConfigurationKeys.AUTO_OFFSET_RESET_CONFIG)); - - consumerConfig.put(ConsumerConfig.GROUP_ID_CONFIG, config - .getString(ConfigurationKeys.APPLICATION_NAME) + "-configuration"); + consumerConfig.put( + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, + config.getString(ConfigurationKeys.AUTO_OFFSET_RESET_CONFIG)); + consumerConfig.put( + ConsumerConfig.GROUP_ID_CONFIG, config + .getString(ConfigurationKeys.APPLICATION_NAME) + "-configuration"); return consumerConfig; } diff --git a/theodolite-benchmarks/uc4-beam/src/main/java/application/UpdateChildParentPairs.java b/theodolite-benchmarks/uc4-beam/src/main/java/application/UpdateChildParentPairs.java index 8692be5ae6637ebda86f10d66b43c6071264e099..cff04e132a93f6c8098c3039232dd48084e6d264 100644 --- a/theodolite-benchmarks/uc4-beam/src/main/java/application/UpdateChildParentPairs.java +++ b/theodolite-benchmarks/uc4-beam/src/main/java/application/UpdateChildParentPairs.java @@ -12,11 +12,12 @@ import org.apache.beam.sdk.values.KV; */ public class UpdateChildParentPairs extends DoFn<KV<String, Set<String>>, KV<String, Set<String>>> { + private static final String STATE_STORE_NAME = "UpdateParents"; + private static final long serialVersionUID = 1L; - @StateId("parents") - private final StateSpec<ValueState<Set<String>>> parents = // NOPMD - StateSpecs.value(); + @StateId(STATE_STORE_NAME) + private final StateSpec<ValueState<Set<String>>> parents = StateSpecs.value(); // NOPMD /** * Match the changes accordingly. @@ -24,9 +25,10 @@ public class UpdateChildParentPairs extends DoFn<KV<String, Set<String>>, KV<Str * @param kv the sensor parents set that contains the changes. */ @ProcessElement - public void processElement(@Element final KV<String, Set<String>> kv, + public void processElement( + @Element final KV<String, Set<String>> kv, final OutputReceiver<KV<String, Set<String>>> out, - @StateId("parents") final ValueState<Set<String>> state) { + @StateId(STATE_STORE_NAME) final ValueState<Set<String>> state) { if (kv.getValue() == null || !kv.getValue().equals(state.read())) { out.output(kv); state.write(kv.getValue()); diff --git a/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordCoder.java b/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordCoder.java index d2b484f5ab30be63f311d6dbcf495baebbd5e2b4..3e0be0fa456efa3ec67504ea9d0e285ae8b3b913 100644 --- a/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordCoder.java +++ b/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordCoder.java @@ -11,8 +11,7 @@ import org.apache.beam.sdk.coders.CoderException; import titan.ccp.model.records.AggregatedActivePowerRecord; /** - * Wrapper Class that encapsulates a AggregatedActivePowerRecord Serde in a - * org.apache.beam.sdk.coders.Coder. + * {@link Coder} for an {@link AggregatedActivePowerRecord}. */ @SuppressWarnings("serial") public class AggregatedActivePowerRecordCoder extends Coder<AggregatedActivePowerRecord> @@ -45,13 +44,13 @@ public class AggregatedActivePowerRecordCoder extends Coder<AggregatedActivePowe @Override public List<? extends Coder<?>> getCoderArguments() { - return null; + return List.of(); } @Override public void verifyDeterministic() throws NonDeterministicException { if (!DETERMINISTIC) { - throw new NonDeterministicException(this, "This class should be deterministic!"); + throw new NonDeterministicException(this, "This class should be deterministic."); } } } diff --git a/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordDeserializer.java b/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordDeserializer.java index 6e2f2765ff65d3bca2a127be36db0854f15afebc..3076861a53dac031afd9e8eb913b5a0bafe480c0 100644 --- a/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordDeserializer.java +++ b/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordDeserializer.java @@ -1,34 +1,12 @@ package serialization; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import org.apache.beam.sdk.coders.AvroCoder; +import io.confluent.kafka.streams.serdes.avro.SpecificAvroDeserializer; import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import titan.ccp.model.records.AggregatedActivePowerRecord; /** - * Wrapper Class that encapsulates a IMonitoringRecordSerde.serializer in a Deserializer + * {@link Deserializer} for an {@link AggregatedActivePowerRecord}. */ public class AggregatedActivePowerRecordDeserializer - implements Deserializer<AggregatedActivePowerRecord> { - - private static final Logger LOGGER = - LoggerFactory.getLogger(AggregatedActivePowerRecordDeserializer.class); - - private final transient AvroCoder<AggregatedActivePowerRecord> avroEnCoder = - AvroCoder.of(AggregatedActivePowerRecord.class); - - @Override - public AggregatedActivePowerRecord deserialize(final String topic, final byte[] data) { - AggregatedActivePowerRecord value = null; - try { - value = this.avroEnCoder.decode(new ByteArrayInputStream(data)); - } catch (final IOException e) { - LOGGER.error("Could not deserialize AggregatedActivePowerRecord", e); - } - return value; - } - + extends SpecificAvroDeserializer<AggregatedActivePowerRecord> { } diff --git a/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordSerializer.java b/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordSerializer.java index 77b79d5465f1d561870bf5b04f8fa20f87076adb..26801d8a28b9756214c65c4e8190e15d04bb3e68 100644 --- a/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordSerializer.java +++ b/theodolite-benchmarks/uc4-beam/src/main/java/serialization/AggregatedActivePowerRecordSerializer.java @@ -1,45 +1,12 @@ package serialization; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import org.apache.beam.sdk.coders.AvroCoder; +import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerializer; import org.apache.kafka.common.serialization.Serializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import titan.ccp.model.records.AggregatedActivePowerRecord; /** - * Wrapper Class that encapsulates a IMonitoringRecordSerde.serializer in a Serializer + * {@link Serializer} for an {@link AggregatedActivePowerRecord}. */ public class AggregatedActivePowerRecordSerializer - implements Serializer<AggregatedActivePowerRecord> { - - private static final Logger LOGGER = - LoggerFactory.getLogger(AggregatedActivePowerRecordSerializer.class); - - private final transient AvroCoder<AggregatedActivePowerRecord> avroEnCoder = - AvroCoder.of(AggregatedActivePowerRecord.class); - - @Override - public byte[] serialize(final String topic, final AggregatedActivePowerRecord data) { - final ByteArrayOutputStream out = new ByteArrayOutputStream(); - try { - this.avroEnCoder.encode(data, out); - } catch (final IOException e) { - LOGGER.error("Could not serialize AggregatedActivePowerRecord", e); - } - final byte[] result = out.toByteArray(); - try { - out.close(); - } catch (final IOException e) { - LOGGER.error( - "Could not close output stream after serialization of AggregatedActivePowerRecord", e); - } - return result; - } - - @Override - public void close() { - Serializer.super.close(); - } + extends SpecificAvroSerializer<AggregatedActivePowerRecord> { } diff --git a/theodolite-benchmarks/uc4-flink/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc4-flink/.settings/org.eclipse.jdt.ui.prefs index 272e01533f6a345d53d2635c47e38c6d3c33dc8a..08fcb07933ca19165976bffd5e7fdfdaf64ee1d2 100644 --- a/theodolite-benchmarks/uc4-flink/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc4-flink/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc4-flink/Dockerfile b/theodolite-benchmarks/uc4-flink/Dockerfile index 4f51f379e5da436104bb8c914e3233d6ecb4ec1f..1fe58e9930db0ef30fa10d12519dd68ccdb2e97e 100644 --- a/theodolite-benchmarks/uc4-flink/Dockerfile +++ b/theodolite-benchmarks/uc4-flink/Dockerfile @@ -1,3 +1,3 @@ -FROM flink:1.12-scala_2.12-java11 +FROM flink:1.13-java11 ADD build/libs/uc4-flink-all.jar /opt/flink/usrlib/artifacts/uc4-flink-all.jar \ No newline at end of file diff --git a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java index 3e2878a893057024de00333492462f5029eb6d77..48eadff101905b746ab239f6030decb728c12475 100644 --- a/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java +++ b/theodolite-benchmarks/uc4-flink/src/main/java/theodolite/uc4/application/AggregationServiceFlinkJob.java @@ -9,12 +9,12 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.state.StateBackend; -import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; +import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; import org.apache.kafka.common.serialization.Serdes; import org.slf4j.Logger; @@ -68,8 +68,6 @@ public final class AggregationServiceFlinkJob { } private void configureEnv() { - this.env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); - final boolean checkpointing = this.config.getBoolean(ConfigurationKeys.CHECKPOINTING, true); final int commitIntervalMs = this.config.getInt(ConfigurationKeys.COMMIT_INTERVAL_MS); if (checkpointing) { @@ -103,9 +101,11 @@ public final class AggregationServiceFlinkJob { this.env.getConfig().registerTypeWithKryoSerializer(Set.of(1, 2, 3, 4).getClass(), // NOCS new ImmutableSetSerializer()); - this.env.getConfig().getRegisteredTypesWithKryoSerializers() - .forEach((c, s) -> LOGGER.info("Class " + c.getName() + " registered with serializer " - + s.getSerializer().getClass().getName())); + this.env + .getConfig() + .getRegisteredTypesWithKryoSerializers() + .forEach((c, s) -> LOGGER.info("Class '{}' registered with serializer '{}'.", c.getName(), + s.getSerializer().getClass().getName())); } private void buildPipeline() { @@ -134,12 +134,13 @@ public final class AggregationServiceFlinkJob { final FlinkKafkaConsumer<AggregatedActivePowerRecord> kafkaOutputSource = kafkaConnector.createConsumer(outputTopic, AggregatedActivePowerRecord.class); - final FlinkKafkaConsumer<Tuple2<Event, String>> kafkaConfigSource = + final FlinkKafkaConsumerBase<Tuple2<Event, String>> kafkaConfigSource = kafkaConnector.createConsumer( configurationTopic, EventSerde::serde, Serdes::String, - TupleType.of(TypeInformation.of(Event.class), Types.STRING)); + TupleType.of(TypeInformation.of(Event.class), Types.STRING)) + .setStartFromEarliest(); // Sink to output topic with SensorId, AggregatedActivePowerRecord final FlinkKafkaProducer<Tuple2<String, AggregatedActivePowerRecord>> kafkaAggregationSink = diff --git a/theodolite-benchmarks/uc4-kstreams/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc4-kstreams/.settings/org.eclipse.jdt.ui.prefs index fa98ca63d77bdee891150bd6713f70197a75cefc..a375cb792eeb842ecfd1f789fbf6a716df43e9c8 100644 --- a/theodolite-benchmarks/uc4-kstreams/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc4-kstreams/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite-benchmarks/uc4-kstreams/src/main/java/theodolite/uc4/streamprocessing/TopologyBuilder.java b/theodolite-benchmarks/uc4-kstreams/src/main/java/theodolite/uc4/streamprocessing/TopologyBuilder.java index 623870313cd341d0594fee38d2fd0ae297abbeae..712b20cb63c2d9f6b77321eaf18eafe4b16854d2 100644 --- a/theodolite-benchmarks/uc4-kstreams/src/main/java/theodolite/uc4/streamprocessing/TopologyBuilder.java +++ b/theodolite-benchmarks/uc4-kstreams/src/main/java/theodolite/uc4/streamprocessing/TopologyBuilder.java @@ -146,7 +146,7 @@ public class TopologyBuilder { .groupByKey(Grouped.with( SensorParentKeySerde.serde(), this.srAvroSerdeFactory.forValues())) - .windowedBy(TimeWindows.of(this.emitPeriod).grace(this.gracePeriod)) + .windowedBy(TimeWindows.ofSizeAndGrace(this.emitPeriod, this.gracePeriod)) .reduce( // TODO Configurable window aggregation function (oldVal, newVal) -> newVal.getTimestamp() >= oldVal.getTimestamp() ? newVal : oldVal, diff --git a/theodolite-benchmarks/uc4-load-generator/.settings/org.eclipse.jdt.ui.prefs b/theodolite-benchmarks/uc4-load-generator/.settings/org.eclipse.jdt.ui.prefs index 4d01df75552c562406705858b6368ecf59d6e82f..ac23341bf71ac68df4183361493261758fd5dafb 100644 --- a/theodolite-benchmarks/uc4-load-generator/.settings/org.eclipse.jdt.ui.prefs +++ b/theodolite-benchmarks/uc4-load-generator/.settings/org.eclipse.jdt.ui.prefs @@ -61,7 +61,7 @@ cleanup_settings_version=2 eclipse.preferences.version=1 editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true formatter_profile=_CAU-SE-Style -formatter_settings_version=15 +formatter_settings_version=21 org.eclipse.jdt.ui.ignorelowercasenames=true org.eclipse.jdt.ui.importorder=; org.eclipse.jdt.ui.ondemandthreshold=99 diff --git a/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt b/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt index 5bd39dcbce40a833fea05b68a86e13e7a1836d3a..f85b83497e5d69e43c1d4784ef86170a5436e929 100644 --- a/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt +++ b/theodolite/src/main/kotlin/theodolite/benchmark/ConfigMapResourceSet.kt @@ -32,11 +32,11 @@ class ConfigMapResourceSet : ResourceSet, KubernetesResource { } if (::files.isInitialized) { - resources = resources.filter { files.contains(it.key) } - - if (resources.size != files.size) { + val filteredResources = resources.filter { files.contains(it.key) } + if (filteredResources.size != files.size) { throw DeploymentFailedException("Could not find all specified Kubernetes manifests files") } + resources = filteredResources } return try { diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt index 871b8cf43907fcb8b0b5ea501c6b47f82e56ff69..36cfef9ce912886a638c200b502923dfe03ef5d0 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/AbstractK8sLoader.kt @@ -13,6 +13,7 @@ abstract class AbstractK8sLoader: K8sResourceLoader { "Deployment" -> loadDeployment(resourceString) "Service" -> loadService(resourceString) "ServiceMonitor" -> loadServiceMonitor(resourceString) + "PodMonitor" -> loadPodMonitor(resourceString) "ConfigMap" -> loadConfigmap(resourceString) "StatefulSet" -> loadStatefulSet(resourceString) "Execution" -> loadExecution(resourceString) @@ -51,6 +52,16 @@ abstract class AbstractK8sLoader: K8sResourceLoader { return loadCustomResourceWrapper(resource, context) } + override fun loadPodMonitor(resource: String): KubernetesResource { + val context = K8sContextFactory().create( + api = "v1", + scope = "Namespaced", + group = "monitoring.coreos.com", + plural = "podmonitors" + ) + return loadCustomResourceWrapper(resource, context) + } + override fun loadExecution(resource: String): KubernetesResource { val context = K8sContextFactory().create( api = "v1", diff --git a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt index c123ab2958132cb43ad188136f738b561e91310b..1487b64bf4f7fbcc735539a429be9237d41205bc 100644 --- a/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt +++ b/theodolite/src/main/kotlin/theodolite/k8s/resourceLoader/K8sResourceLoader.kt @@ -11,5 +11,6 @@ interface K8sResourceLoader { fun loadBenchmark(resource: String): KubernetesResource fun loadConfigmap(resource: String): KubernetesResource fun loadServiceMonitor(resource: String): KubernetesResource + fun loadPodMonitor(resource: String): KubernetesResource fun loadCustomResourceWrapper(resource: String, context: CustomResourceDefinitionContext): KubernetesResource } \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/patcher/MatchLabelPatcher.kt b/theodolite/src/main/kotlin/theodolite/patcher/MatchLabelPatcher.kt new file mode 100644 index 0000000000000000000000000000000000000000..30ff73b5da3b551119ad085adbc982180e4fc066 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/patcher/MatchLabelPatcher.kt @@ -0,0 +1,34 @@ +package theodolite.patcher + +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.api.model.apps.Deployment +import io.fabric8.kubernetes.api.model.apps.StatefulSet + +/** + * This patcher is able to set the `spec.selector.matchLabels` for a `Deployment` or `StatefulSet` Kubernetes resource. + * + * @property k8sResource The Kubernetes manifests to patch + * @property variableName The matchLabel which should be set + */ +class MatchLabelPatcher(private val k8sResource: KubernetesResource, val variableName: String) : + AbstractPatcher(k8sResource) { + + override fun <String> patch(labelValue: String) { + if (labelValue is kotlin.String) { + when (k8sResource) { + is Deployment -> { + if (k8sResource.spec.selector.matchLabels == null) { + k8sResource.spec.selector.matchLabels = mutableMapOf() + } + k8sResource.spec.selector.matchLabels[this.variableName] = labelValue + } + is StatefulSet -> { + if (k8sResource.spec.selector.matchLabels == null) { + k8sResource.spec.selector.matchLabels = mutableMapOf() + } + k8sResource.spec.selector.matchLabels[this.variableName] = labelValue + } + } + } + } +} \ No newline at end of file diff --git a/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt b/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt index 88b3e19e999a889cdcb8345ca7c90c37a6e6d275..e92de4dba7de298c9df76600f2c6785f5878103e 100644 --- a/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt +++ b/theodolite/src/main/kotlin/theodolite/patcher/PatcherFactory.kt @@ -79,6 +79,14 @@ class PatcherFactory { k8sResource = resource, variableName = patcherDefinition.properties["variableName"]!! ) + "MatchLabelPatcher" -> MatchLabelPatcher( + k8sResource = resource, + variableName = patcherDefinition.properties["variableName"]!! + ) + "TemplateLabelPatcher" -> TemplateLabelPatcher( + k8sResource = resource, + variableName = patcherDefinition.properties["variableName"]!! + ) "ImagePatcher" -> ImagePatcher( k8sResource = resource, container = patcherDefinition.properties["container"]!! diff --git a/theodolite/src/main/kotlin/theodolite/patcher/TemplateLabelPatcher.kt b/theodolite/src/main/kotlin/theodolite/patcher/TemplateLabelPatcher.kt new file mode 100644 index 0000000000000000000000000000000000000000..a524e5c40f90ccf98dc95003cc33dcfceb6f8598 --- /dev/null +++ b/theodolite/src/main/kotlin/theodolite/patcher/TemplateLabelPatcher.kt @@ -0,0 +1,34 @@ +package theodolite.patcher + +import io.fabric8.kubernetes.api.model.KubernetesResource +import io.fabric8.kubernetes.api.model.apps.Deployment +import io.fabric8.kubernetes.api.model.apps.StatefulSet + +/** + * This patcher is able to set the field `spec.template.metadata.labels` for a `Deployment` or `StatefulSet` Kubernetes resource. + * + * @property k8sResource The Kubernetes manifests to patch + * @property variableName The label which should be set + */ +class TemplateLabelPatcher(private val k8sResource: KubernetesResource, val variableName: String) : + AbstractPatcher(k8sResource) { + + override fun <String> patch(labelValue: String) { + if (labelValue is kotlin.String) { + when (k8sResource) { + is Deployment -> { + if (k8sResource.spec.template.metadata.labels == null) { + k8sResource.spec.template.metadata.labels = mutableMapOf() + } + k8sResource.spec.template.metadata.labels[this.variableName] = labelValue + } + is StatefulSet -> { + if (k8sResource.spec.template.metadata.labels == null) { + k8sResource.spec.template.metadata.labels = mutableMapOf() + } + k8sResource.spec.template.metadata.labels[this.variableName] = labelValue + } + } + } + } +} \ No newline at end of file