Compare revisions

Sören Henning · Sören Henning · Sören Henning · Sören Henning · Björn Vonheiden · Sören Henning
--- a/benchmarks/.gitlab-ci.yml
+++ b/benchmarks/.gitlab-ci.yml
@@ -243,3 +243,172 @@ deploy-uc4-workload-generator:
    - if: "$DOCKERHUB_ORG && $DOCKERHUB_ID && $DOCKERHUB_PW && $IMAGE_NAME && $JAVA_PROJECT_NAME"
      when: manual
      allow_failure: true
+
+.deploy-ghcr:
+  stage: deploy
+  tags:
+    - exec-dind
+  # see https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled
+  # for image usage and settings for building with TLS and docker in docker
+  image: docker:19.03.1
+  services:
+    - docker:19.03.1-dind
+  variables:
+    DOCKER_TLS_CERTDIR: "/certs"
+  script:
+    - DOCKER_TAG_NAME=$(echo $CI_COMMIT_REF_SLUG- | sed 's/^master-$//')
+    - docker build --pull -t $IMAGE_NAME ./$JAVA_PROJECT_NAME
+    - "[ ! $CI_COMMIT_TAG ] && docker tag $IMAGE_NAME ghcr.io/$GITHUB_CR_ORG/$IMAGE_NAME:${DOCKER_TAG_NAME}latest"
+    - "[ ! $CI_COMMIT_TAG ] && docker tag $IMAGE_NAME ghcr.io/$GITHUB_CR_ORG/$IMAGE_NAME:$DOCKER_TAG_NAME$CI_COMMIT_SHORT_SHA"
+    - "[ $CI_COMMIT_TAG ] && docker tag $IMAGE_NAME ghcr.io/$GITHUB_CR_ORG/$IMAGE_NAME:$CI_COMMIT_TAG"
+    - echo $GITHUB_CR_TOKEN | docker login ghcr.io -u $GITHUB_CR_USER --password-stdin
+    - docker push ghcr.io/$GITHUB_CR_ORG/$IMAGE_NAME
+    - docker logout
+  rules:
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME && $CI_COMMIT_TAG"
+      when: always
+    - changes:
+      # - $JAVA_PROJECT_NAME/**/* # hope this can be simplified soon, see #51
+      - benchmarks/application-kafkastreams-commons/**/*
+      if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: always
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: manual
+      allow_failure: true
+
+deploy-ghcr-uc1-kstreams-app:
+  extends: .deploy-ghcr
+  variables:
+    IMAGE_NAME: "theodolite-uc1-kstreams-app"
+    JAVA_PROJECT_NAME: "uc1-application"
+  rules: # hope this can be simplified soon, see #51
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME && $CI_COMMIT_TAG"
+      when: always
+    - changes:
+      - benchmarks/uc1-application/**/*
+      - benchmarks/application-kafkastreams-commons/**/*
+      if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: always
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: manual
+      allow_failure: true
+
+deploy-ghcr-uc2-kstreams-app:
+  extends: .deploy-ghcr
+  variables:
+    IMAGE_NAME: "theodolite-uc2-kstreams-app"
+    JAVA_PROJECT_NAME: "uc2-application"
+  rules: # hope this can be simplified soon, see #51
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME && $CI_COMMIT_TAG"
+      when: always
+    - changes:
+      - benchmarks/uc2-application/**/*
+      - benchmarks/application-kafkastreams-commons/**/*
+      if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: always
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: manual
+      allow_failure: true
+
+deploy-ghcr-uc3-kstreams-app:
+  extends: .deploy-ghcr
+  variables:
+    IMAGE_NAME: "theodolite-uc3-kstreams-app"
+    JAVA_PROJECT_NAME: "uc3-application"
+  rules: # hope this can be simplified soon, see #51
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME && $CI_COMMIT_TAG"
+      when: always
+    - changes:
+      - benchmarks/uc3-application/**/*
+      - benchmarks/application-kafkastreams-commons/**/*
+      if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: always
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: manual
+      allow_failure: true
+
+deploy-ghcr-uc4-kstreams-app:
+  extends: .deploy-ghcr
+  variables:
+    IMAGE_NAME: "theodolite-uc4-kstreams-app"
+    JAVA_PROJECT_NAME: "uc4-application"
+  rules: # hope this can be simplified soon, see #51
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME && $CI_COMMIT_TAG"
+      when: always
+    - changes:
+      - benchmarks/uc4-application/**/*
+      - benchmarks/application-kafkastreams-commons/**/*
+      if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: always
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: manual
+      allow_failure: true
+
+deploy-ghcr-uc1-workload-generator:
+  extends: .deploy-ghcr
+  variables:
+    IMAGE_NAME: "theodolite-uc1-workload-generator"
+    JAVA_PROJECT_NAME: "uc1-workload-generator"
+  rules: # hope this can be simplified soon, see #51
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME && $CI_COMMIT_TAG"
+      when: always
+    - changes:
+      - benchmarks/uc1-workload-generator/**/*
+      - benchmarks/application-kafkastreams-commons/**/*
+      if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: always
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: manual
+      allow_failure: true
+
+deploy-ghcr-uc2-workload-generator:
+  extends: .deploy-ghcr
+  variables:
+    IMAGE_NAME: "theodolite-uc2-workload-generator"
+    JAVA_PROJECT_NAME: "uc2-workload-generator"
+  rules: # hope this can be simplified soon, see #51
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME && $CI_COMMIT_TAG"
+      when: always
+    - changes:
+      - benchmarks/uc2-workload-generator/**/*
+      - benchmarks/application-kafkastreams-commons/**/*
+      if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: always
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: manual
+      allow_failure: true
+
+deploy-ghcr-uc3-workload-generator:
+  extends: .deploy-ghcr
+  variables:
+    IMAGE_NAME: "theodolite-uc3-workload-generator"
+    JAVA_PROJECT_NAME: "uc3-workload-generator"
+  rules: # hope this can be simplified soon, see #51
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME && $CI_COMMIT_TAG"
+      when: always
+    - changes:
+      - benchmarks/uc3-workload-generator/**/*
+      - benchmarks/application-kafkastreams-commons/**/*
+      if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: always
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: manual
+      allow_failure: true
+
+deploy-ghcr-uc4-workload-generator:
+  extends: .deploy-ghcr
+  variables:
+    IMAGE_NAME: "theodolite-uc4-workload-generator"
+    JAVA_PROJECT_NAME: "uc4-workload-generator"
+  rules: # hope this can be simplified soon, see #51
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME && $CI_COMMIT_TAG"
+      when: always
+    - changes:
+      - benchmarks/uc4-workload-generator/**/*
+      - benchmarks/application-kafkastreams-commons/**/*
+      if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: always
+    - if: "$GITHUB_CR_ORG && $GITHUB_CR_USER && $GITHUB_CR_TOKEN && $IMAGE_NAME && $JAVA_PROJECT_NAME"
+      when: manual
+      allow_failure: true
+
--- a/.settings/org.eclipse.jdt.ui.prefs
+++ b/.settings/org.eclipse.jdt.ui.prefs
--- a/.settings/qa.eclipse.plugin.checkstyle.prefs
+++ b/.settings/qa.eclipse.plugin.checkstyle.prefs
--- a/.settings/qa.eclipse.plugin.pmd.prefs
+++ b/.settings/qa.eclipse.plugin.pmd.prefs
--- a/execution/run_uc.py
+++ b/execution/run_uc.py
@@ -134,14 +134,15 @@ def start_workload_generator(wg_yaml, dim_value, uc_id):
        num_nested_groups = dim_value
        num_sensors = 4
        approx_num_sensors = num_sensors ** num_nested_groups
-        wl_instances = (approx_num_sensors + wl_max_records - 1) // wl_max_records
+        wl_instances = (approx_num_sensors +
+                        wl_max_records - 1) // wl_max_records

    # Customize workload generator creations
    wg_yaml['spec']['replicas'] = wl_instances
    # Set used use case
    wg_containter = next(filter(
        lambda x: x['name'] == 'workload-generator', wg_yaml['spec']['template']['spec']['containers']))
-    wg_containter['image'] = 'theodolite/theodolite-uc' + uc_id + \
+    wg_containter['image'] = 'ghcr.io/cau-se/theodolite-uc' + uc_id + \
        '-workload-generator:latest'
    # Set environment variables

@@ -224,7 +225,7 @@ def start_application(svc_yaml, svc_monitor_yaml, jmx_yaml, deploy_yaml,
    app_container = next(filter(
        lambda x: x['name'] == 'uc-application',
        deploy_yaml['spec']['template']['spec']['containers']))
-    app_container['image'] = 'theodolite/theodolite-uc' + uc_id \
+    app_container['image'] = 'ghcr.io/cau-se/theodolite-uc' + uc_id \
        + '-kstreams-app:latest'

    # Set configurations environment parameters for SPE
@@ -341,6 +342,24 @@ def stop_applications(wg, app_svc, app_svc_monitor, app_jmx, app_deploy):

    print('Delete uc application')
    delete_resource(app_deploy, appsApi.delete_namespaced_deployment)
+
+    print('Check all pods deleted.')
+    while True:
+        # Wait bit for deletion
+        time.sleep(2)
+
+        # Count how many pod still need to be deleted
+        no_load = len(coreApi.list_namespaced_pod(
+            namespace, label_selector='app=titan-ccp-load-generator').items)
+        no_uc = len(coreApi.list_namespaced_pod(
+            namespace, label_selector='app=titan-ccp-aggregation').items)
+
+        # Check if all pods deleted
+        if no_load <= 0 and no_uc <= 0:
+            print('All pods deleted.')
+            break
+
+        print(f'#{no_load} load generator and #{no_uc} uc pods needs to be deleted')
    return



--- a/execution/theodolite.py
+++ b/execution/theodolite.py
@@ -69,7 +69,7 @@ def main(uc, loads, instances_list, partitions, cpu_limit, memory_limit,
        write_stream.write(str(exp_id + 1))

    domain_restriction_strategy = None
-    search_strategy = None
+    search_strategy_method = None

    # Select domain restriction
    if domain_restriction:
@@ -83,13 +83,13 @@ def main(uc, loads, instances_list, partitions, cpu_limit, memory_limit,
    if search_strategy == "linear-search":
        print(
            f"Going to execute at most {len(loads)+len(instances_list)-1} subexperiments in total..")
-        search_strategy = linear_search_strategy
+        search_strategy_method = linear_search_strategy
    elif search_strategy == "binary-search":
-        search_strategy = binary_search_strategy
+        search_strategy_method = binary_search_strategy
    else:
        print(
            f"Going to execute {len(loads)*len(instances_list)} subexperiments in total..")
-        search_strategy = check_all_strategy
+        search_strategy_method = check_all_strategy

    experiment_config = ExperimentConfig(
        use_case=uc,
@@ -106,7 +106,7 @@ def main(uc, loads, instances_list, partitions, cpu_limit, memory_limit,
        configurations=configurations,
        result_path=result_path,
        domain_restriction_strategy=domain_restriction_strategy,
-        search_strategy=search_strategy,
+        search_strategy=search_strategy_method,
        subexperiment_executor=subexperiment_executor,
        subexperiment_evaluator=subexperiment_evaluator)
No results found