From 3e6e3b9db45f6e94cbebc6363621420c7e83ad5d Mon Sep 17 00:00:00 2001 From: Dagfinn Olsen Date: Thu, 13 Feb 2025 10:39:35 +0100 Subject: [PATCH] chore(performance): Adjust thresholds (#1870) ## Description Adjust smoketests thresholds for p95 and removed thresholds for p99. Use k8s k6 operator cluster instead of github runner ## Related Issue(s) - #1862 ## Verification - [ ] **Your** code builds clean without any errors or warnings - [ ] Manual testing done (required) - [ ] Relevant automated test added (if you find this hard, leave it and we'll help out) ## Documentation - [ ] Documentation is updated (either in `docs`-directory, Altinnpedia or a separate linked PR in [altinn-studio-docs.](https://github.com/Altinn/altinn-studio-docs), if applicable) --- .github/workflows/ci-cd-yt01.yml | 9 ++++--- .../workflows/workflow-run-k6-performance.yml | 7 ++++++ .../enduserSearchWithThresholds.js | 18 ++++++------- .../performance/createDialogWithThresholds.js | 14 +++++++---- .../createTransmissionsWithThresholds.js | 25 +++++++++++++++++++ .../serviceOwnerSearchWithThresholds.js | 16 ++++++------ 6 files changed, 64 insertions(+), 25 deletions(-) create mode 100644 tests/k6/tests/serviceowner/performance/createTransmissionsWithThresholds.js diff --git a/.github/workflows/ci-cd-yt01.yml b/.github/workflows/ci-cd-yt01.yml index 7ff6f1eec..5c8d78740 100644 --- a/.github/workflows/ci-cd-yt01.yml +++ b/.github/workflows/ci-cd-yt01.yml @@ -115,10 +115,10 @@ jobs: if: ${{ always() && !failure() && !cancelled() && (needs.check-for-changes.outputs.hasBackendChanges == 'true' || needs.check-for-changes.outputs.hasInfraChanges == 'true') }} needs: [deploy-apps, deploy-infra, check-for-changes] #needs: [deploy-apps, check-for-changes] - uses: ./.github/workflows/workflow-run-k6-ci-cd-yt01.yml + uses: ./.github/workflows/workflow-run-k6-performance.yml secrets: - TOKEN_GENERATOR_USERNAME: ${{ secrets.TOKEN_GENERATOR_USERNAME }} - TOKEN_GENERATOR_PASSWORD: ${{ secrets.TOKEN_GENERATOR_PASSWORD }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} strategy: max-parallel: 1 matrix: @@ -133,6 +133,9 @@ jobs: vus: 1 duration: 30s testSuitePath: ${{ matrix.files }} + parallelism: 1 + breakpoint: false + abortOnFail: false ref: "refs/tags/v${{ github.event.client_payload.version }}" permissions: checks: write diff --git a/.github/workflows/workflow-run-k6-performance.yml b/.github/workflows/workflow-run-k6-performance.yml index 138b4671c..3d6aa3da7 100644 --- a/.github/workflows/workflow-run-k6-performance.yml +++ b/.github/workflows/workflow-run-k6-performance.yml @@ -27,6 +27,11 @@ on: abortOnFail: required: true type: boolean + ref: + description: "The branch or tag ref to run the tests on. Using default checkout ref if not provided." + required: false + default: ${{ github.ref }} + type: string secrets: AZURE_CLIENT_ID: required: true @@ -42,6 +47,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} - name: OIDC Login to Azure Public Cloud uses: azure/login@v2 with: diff --git a/tests/k6/tests/enduser/performance/enduserSearchWithThresholds.js b/tests/k6/tests/enduser/performance/enduserSearchWithThresholds.js index 4cfc611bf..bf03ab17b 100644 --- a/tests/k6/tests/enduser/performance/enduserSearchWithThresholds.js +++ b/tests/k6/tests/enduser/performance/enduserSearchWithThresholds.js @@ -7,15 +7,15 @@ export let options = { duration: "30s", thresholds: { ..._options.thresholds, - "http_req_duration{name:enduser search}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get dialog}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get dialog activities}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get dialog activity}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get seenlogs}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get seenlog}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get transmissions}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get transmission}": ["p(95)<300", "p(99)<500"], - "http_req_duration{name:get labellog}": ["p(95)<300", "p(99)<500"] + "http_req_duration{name:enduser search}": ["p(95)<300"], + "http_req_duration{name:get dialog}": ["p(95)<300"], + "http_req_duration{name:get dialog activities}": ["p(95)<300"], + "http_req_duration{name:get dialog activity}": ["p(95)<300"], + "http_req_duration{name:get seenlogs}": ["p(95)<300"], + "http_req_duration{name:get seenlog}": ["p(95)<300"], + "http_req_duration{name:get transmissions}": ["p(95)<300"], + "http_req_duration{name:get transmission}": ["p(95)<300"], + "http_req_duration{name:get labellog}": ["p(95)<300"] } }; diff --git a/tests/k6/tests/serviceowner/performance/createDialogWithThresholds.js b/tests/k6/tests/serviceowner/performance/createDialogWithThresholds.js index 01803e376..53fc5bce2 100644 --- a/tests/k6/tests/serviceowner/performance/createDialogWithThresholds.js +++ b/tests/k6/tests/serviceowner/performance/createDialogWithThresholds.js @@ -1,14 +1,18 @@ -import { default as run } from "./create-dialog.js"; -export { setup as setup } from '../../performancetest_common/readTestdata.js'; - +import { randomItem } from '../../../common/k6-utils.js'; +import { createDialog } from '../../performancetest_common/createDialog.js'; +import { serviceOwners, endUsers } from '../../performancetest_common/readTestdata.js'; export let options = { summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'p(99.5)', 'p(99.9)', 'count'], vus: 1, duration: "30s", thresholds: { - "http_req_duration{name:create dialog}": ["p(95)<300", "p(99)<500"], + "http_req_duration{name:create dialog}": ["p(95)<200"], "http_reqs{name:create dialog}": [] } } -export default function (data) { run(data); } \ No newline at end of file +const traceCalls = (__ENV.traceCalls ?? 'false') === 'true'; + +export default function () { + createDialog(serviceOwners[0], randomItem(endUsers), traceCalls); +} \ No newline at end of file diff --git a/tests/k6/tests/serviceowner/performance/createTransmissionsWithThresholds.js b/tests/k6/tests/serviceowner/performance/createTransmissionsWithThresholds.js new file mode 100644 index 000000000..1fa0771b1 --- /dev/null +++ b/tests/k6/tests/serviceowner/performance/createTransmissionsWithThresholds.js @@ -0,0 +1,25 @@ +import { randomItem } from '../../../common/k6-utils.js'; +import { createTransmissions } from '../../performancetest_common/createDialog.js'; +import { serviceOwners, endUsers } from '../../performancetest_common/readTestdata.js'; + +export const options = { + summaryTrendStats: ['avg', 'min', 'med', 'max', 'p(95)', 'p(99)', 'p(99.5)', 'p(99.9)', 'count'], + vus: 1, + duration: "30s", + thresholds: { + "http_req_duration{name:create dialog}": ["p(95)<200"], + "http_reqs{name:create dialog}": [], + "http_req_duration{name:create transmission}": ["p(95)<200"], + "http_reqs{name:create transmission}": [] + } +}; + +const traceCalls = (__ENV.traceCalls ?? 'false') === 'true'; +const numberOfTransmissions = (__ENV.numberOfTransmissions ?? '10'); +const maxTransmissionsInThread = (__ENV.maxTransmissionsInThread ?? '100'); +const testid = (__ENV.TESTID ?? 'createTransmissions'); + +export default function() { + createTransmissions(serviceOwners[0], randomItem(endUsers), traceCalls, numberOfTransmissions, maxTransmissionsInThread, testid); +} + diff --git a/tests/k6/tests/serviceowner/performance/serviceOwnerSearchWithThresholds.js b/tests/k6/tests/serviceowner/performance/serviceOwnerSearchWithThresholds.js index 8b0c3d04b..bf6f9ce8b 100644 --- a/tests/k6/tests/serviceowner/performance/serviceOwnerSearchWithThresholds.js +++ b/tests/k6/tests/serviceowner/performance/serviceOwnerSearchWithThresholds.js @@ -7,14 +7,14 @@ export let options = { duration: "30s", thresholds: { ..._options.thresholds, - "http_req_duration{name:serviceowner search}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get dialog}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get dialog activities}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get dialog activity}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get seenlogs}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get seenlog}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get transmissions}": ["p(95)<100", "p(99)<300"], - "http_req_duration{name:get transmission}": ["p(95)<100", "p(99)<300"], + "http_req_duration{name:serviceowner search}": ["p(95)<300"], + "http_req_duration{name:get dialog}": ["p(95)<300"], + "http_req_duration{name:get dialog activities}": ["p(95)<300"], + "http_req_duration{name:get dialog activity}": ["p(95)<300"], + "http_req_duration{name:get seenlogs}": ["p(95)<300"], + "http_req_duration{name:get seenlog}": ["p(95)<300"], + "http_req_duration{name:get transmissions}": ["p(95)<300"], + "http_req_duration{name:get transmission}": ["p(95)<300"], } }