diff --git a/.github/workflows/performance.yaml b/.github/workflows/performance.yaml index 73d5e38e..4574f3b6 100644 --- a/.github/workflows/performance.yaml +++ b/.github/workflows/performance.yaml @@ -77,6 +77,7 @@ jobs: TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports TEST_METRICS_DIR: ${{ github.workspace }}/test/performance/results/head TEST_RUN_NAME: head + TEST_KUBE_BURNER_ITERATIONS: 15 run: | cd test/performance mkdir -p ./results/head @@ -89,6 +90,7 @@ jobs: TEST_INSPECTION_REPORTS_DIR: ${{ github.workspace }}/inspection-reports TEST_METRICS_DIR: ${{ github.workspace }}/test/performance/results/base-code TEST_RUN_NAME: base-code + TEST_KUBE_BURNER_ITERATIONS: 15 run: | cd test/performance mkdir -p ./results/base-code @@ -114,12 +116,12 @@ jobs: name: performance-results path: ${{ github.workspace }}/test/performance/results - name: Prepare inspection reports - if: failure() + if: always() run: | tar -czvf inspection-reports.tar.gz -C ${{ github.workspace }} inspection-reports echo "artifact_name=inspection-reports" | sed 's/:/-/g' >> $GITHUB_ENV - name: Upload inspection report artifact - if: failure() + if: always() uses: actions/upload-artifact@v4 with: name: ${{ env.artifact_name }} diff --git a/test/performance/tests/test_util/config.py b/test/performance/tests/test_util/config.py index 22d2526b..04c42f78 100644 --- a/test/performance/tests/test_util/config.py +++ b/test/performance/tests/test_util/config.py @@ -39,9 +39,10 @@ os.getenv("TEST_KUBE_BURNER_URL") or "https://github.com/kube-burner/kube-burner/releases/download/v1.2/kube-burner-1.2-Linux-x86_64.tar.gz" ) - # Global kube-burner invocation timeout. KUBE_BURNER_TIMEOUT = os.getenv("TEST_KUBE_BURNER_TIMEOUT") or "10m" +# The number of kube-burner invocations. +KUBE_BURNER_ITERATIONS = int(os.getenv("TEST_KUBE_BURNER_ITERATIONS") or 1) # FLAVOR is the flavour to use for running the performance tests. FLAVOR = os.getenv("TEST_FLAVOR") or "" diff --git a/test/performance/tests/test_util/metrics.py b/test/performance/tests/test_util/metrics.py index 3ba9e88f..8bed5153 100644 --- a/test/performance/tests/test_util/metrics.py +++ b/test/performance/tests/test_util/metrics.py @@ -1,11 +1,14 @@ # # Copyright 2025 Canonical, Ltd. # +import logging import os from typing import List from test_util import config, harness, util +LOG = logging.getLogger(__name__) + def stop_metrics(instances: List[harness.Instance], process_dict: dict): """Stops collecting metrics in the background from each instance.""" @@ -78,17 +81,34 @@ def configure_kube_burner(instance: harness.Instance): ) -def run_kube_burner(instance: harness.Instance): +def run_kube_burner( + instance: harness.Instance, iterations: int = config.KUBE_BURNER_ITERATIONS +): """Copies kubeconfig and runs kube-burner on the instance.""" instance.exec(["mkdir", "-p", "/root/.kube"]) instance.exec(["k8s", "config", ">", "/root/.kube/config"]) - instance.exec( - [ - "/root/kube-burner", - "init", - "--timeout", - config.KUBE_BURNER_TIMEOUT, - "-c", - "/root/api-intensive.yaml", - ] - ) + + raised_exc = None + for iteration in range(iterations): + LOG.info("Staring kube-burner iteration %s of %s.", iteration, iterations) + try: + instance.exec( + [ + "/root/kube-burner", + "init", + "--timeout", + config.KUBE_BURNER_TIMEOUT, + "-c", + "/root/api-intensive.yaml", + ] + ) + except Exception as ex: + # We'll continue the loop even after encountering failures + # in order to determine if this is a transient failure or if the + # dqlite service was completely compromised (e.g. deadlock or crash). + LOG.exception("kube-burner job failed, continuing...") + raised_exc = ex + + # Raise encountered exceptions, if any. + if raised_exc: + raise raised_exc