diff --git a/Makefile b/Makefile index b0fa1d4..3eb4a65 100644 --- a/Makefile +++ b/Makefile @@ -275,6 +275,8 @@ test-locust: pytest port-forward: + @echo "$(shell tput setaf 10)$(shell tput bold)Set up port-forwarding $(shell tput sgr0)" + ./portforwards.sh create-db-mock-secret: create-namespace @@ -297,31 +299,6 @@ deploy-cluster-for-load-testing: create-kindcluster install-prometheus create-db kubectl apply -f ./config/prometheus/grafana-dashboard-mockapi-configmap.yaml run-load-testing: deploy-cluster-for-load-testing deploy-locust port-forward + @echo "$(shell tput setaf 10)$(shell tput bold)Verify load tests $(shell tput sgr0)" - ## - # python3 ./test.py - - sleep 45 - curl localhost:9090 > promstats-locust.txt - - - go get -u github.com/ryotarai/prometheus-query - - prometheus-query -server http://localhost:9091 -query locust_user_count -start "now" -end "now" | jq .[0].values[0].value - - # LOCUST_ARGS?="'--noweb', '-c', '25', '-r', '0.03'" - - # python3 ./test.py ./promstats-lucust.txt - - # Check stats - # while "curl localhost:9090" prom metics "locust_user_count" < 25... wait - - # get prommetrics and extract stats - # check against thresholds - - # pass or fail! - - # prometheus-query -server http://localhost:9091 -query locust_user_count -start "now" -end "now" | jq .[0].values[0].value - # # LOCUST_ARGS?=,'--no-web', '-c', '25', '-r', '0.08 - - + go run verify_load_tests.go \ No newline at end of file diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index b35c87c..e2f7441 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -83,6 +83,10 @@ jobs: failOnStandardError: false inlineScript: az acr build --registry $(ACR_NAME) --image $(IMAGE_NAME) . + - script: make run-load-testing + condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master')) + displayName: 'Run load tests' + - script: | ./bin/kustomize build config/default > $(Build.ArtifactStagingDirectory)/setup.yaml set -x diff --git a/config/prometheus/grafana-dashboard-load-test-configmap.yaml b/config/prometheus/grafana-dashboard-load-test-configmap.yaml index 6b57ca3..ebd1850 100644 --- a/config/prometheus/grafana-dashboard-load-test-configmap.yaml +++ b/config/prometheus/grafana-dashboard-load-test-configmap.yaml @@ -23,7 +23,7 @@ data: "editable": true, "gnetId": null, "graphTooltip": 1, - "iteration": 1576783033807, + "iteration": 1580822557510, "links": [], "panels": [ { @@ -2138,7 +2138,7 @@ data: "x": 0, "y": 54 }, - "id": 50, + "id": 53, "legend": { "avg": false, "current": false, @@ -2226,7 +2226,7 @@ data: "dashLength": 10, "dashes": false, "datasource": null, - "description": "Run duration (as returned by Databricks API)", + "description": "Run duration (from API) and the time from Run start time (as per API) to time the operator reconciles the terminated state and the run duration (from API)", "fill": 1, "fillGradient": 0, "gridPos": { @@ -2235,7 +2235,7 @@ data: "x": 16, "y": 60 }, - "id": 46, + "id": 48, "legend": { "avg": false, "current": false, @@ -2261,16 +2261,21 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(databricks_run_duration_seconds_sum[1m])/rate(databricks_run_duration_seconds_count[1m])", - "legendFormat": "mean_duration-{{life_cycle_state}}", + "expr": "rate(databricks_run_time_to_detect_finished_seconds_sum[1m])/rate(databricks_run_time_to_detect_finished_seconds_count[1m])", + "legendFormat": "time to operator complete - {{life_cycle_state}}", "refId": "A" + }, + { + "expr": "rate(databricks_run_duration_seconds_sum[1m])/rate(databricks_run_duration_seconds_count[1m])", + "legendFormat": "run duration - {{life_cycle_state}}", + "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Run Duration", + "title": "Run duration", "tooltip": { "shared": true, "sort": 0, @@ -2288,7 +2293,7 @@ data: { "decimals": 2, "format": "s", - "label": "", + "label": null, "logBase": 1, "max": null, "min": null, @@ -2314,7 +2319,7 @@ data: "dashLength": 10, "dashes": false, "datasource": null, - "description": "Time from Run start time (as per API) to time the operator reconciles the terminated state", + "description": "Latency in operator.\nDefined as the difference between the time from Run start time (as per API) to time the operator reconciles the terminated state and the run duration (from API)", "fill": 1, "fillGradient": 0, "gridPos": { @@ -2323,7 +2328,7 @@ data: "x": 16, "y": 68 }, - "id": 48, + "id": 52, "legend": { "avg": false, "current": false, @@ -2349,8 +2354,8 @@ data: "steppedLine": false, "targets": [ { - "expr": "rate(databricks_run_time_to_detect_finished_seconds_sum[1m])/rate(databricks_run_time_to_detect_finished_seconds_count[1m])", - "legendFormat": "mean_time-{{life_cycle_state}}", + "expr": "rate(databricks_run_time_to_detect_finished_seconds_sum[1m])/rate(databricks_run_time_to_detect_finished_seconds_count[1m]) - rate(databricks_run_duration_seconds_sum[1m])/rate(databricks_run_duration_seconds_count[1m])", + "legendFormat": "operator latency - {{life_cycle_state}}", "refId": "A" } ], @@ -2358,7 +2363,7 @@ data: "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Time to Run Completion", + "title": "Operator run latency", "tooltip": { "shared": true, "sort": 0, diff --git a/config/prometheus/grafana-dashboard-mockapi-configmap.yaml b/config/prometheus/grafana-dashboard-mockapi-configmap.yaml index af414c4..018d96c 100644 --- a/config/prometheus/grafana-dashboard-mockapi-configmap.yaml +++ b/config/prometheus/grafana-dashboard-mockapi-configmap.yaml @@ -7,518 +7,518 @@ metadata: data: mockapi-dash.json: |- { - "annotations": { - "list": [ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Mean response times for calls to the mock API", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" + "expr": "rate(databricks_mock_api_response_time_sum[1m]) / rate(databricks_mock_api_response_time_count[1m])", + "legendFormat": "{{type}} - {{action}}", + "refId": "A" } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Mean response times for calls to the mock API", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(databricks_mock_api_response_time_sum[1m]) / rate(databricks_mock_api_response_time_count[1m])", - "legendFormat": "{{type}} - {{action}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Response time (mean)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response time (mean)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Response time percentiles for calls to mock API", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Response time percentiles for calls to mock API", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.5, sum(rate(databricks_mock_api_response_time_bucket[1m])) by (le, action, type))", - "legendFormat": "{{type}} - {{action}} - P50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum(rate(databricks_mock_api_response_time_bucket[1m])) by (le, action, type))", - "legendFormat": "{{type}} - {{action}} - P95", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Response time (percentiles)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum(rate(databricks_mock_api_response_time_bucket[1m])) by (le, action, type))", + "legendFormat": "{{type}} - {{action}} - P50", + "refId": "A" }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "expr": "histogram_quantile(0.95, sum(rate(databricks_mock_api_response_time_bucket[1m])) by (le, action, type))", + "legendFormat": "{{type}} - {{action}} - P95", + "refId": "B" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response time (percentiles)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Rate of requests to the mock API (requests per second)", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(databricks_mock_api_response_time_count[1m])) by (type, action, code)", - "hide": false, - "legendFormat": "{{type}} - {{action}} - {{code}}", - "refId": "A" - }, - { - "expr": "sum(rate(databricks_mock_api_response_time_count[1m]))", - "format": "time_series", - "hide": false, - "legendFormat": "Total RPS", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Incoming requests (RPS)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "reqps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Rate of requests to the mock API (requests per second)", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": null, - "description": "Total number of requests to the mock API", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pluginVersion": "6.4.2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "databricks_mock_api_response_time_count", - "legendFormat": "{{type}} - {{action}} - {{code}}", - "refId": "A" - }, - { - "expr": "sum(databricks_mock_api_response_time_count)", - "legendFormat": "total", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Incoming requests (Total)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.4.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(databricks_mock_api_response_time_count[1m])) by (type, action, code)", + "hide": false, + "legendFormat": "{{type}} - {{action}} - {{code}}", + "refId": "A" }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "expr": "sum(rate(databricks_mock_api_response_time_count[1m]))", + "format": "time_series", + "hide": false, + "legendFormat": "Total RPS", + "refId": "B" } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming requests (RPS)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" }, - { - "aliasColors": {}, - "bars": false, - "cacheTimeout": null, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 32 + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Total number of requests to the mock API", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pluginVersion": "6.4.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "databricks_mock_api_response_time_count", + "legendFormat": "{{type}} - {{action}} - {{code}}", + "refId": "A" }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] + { + "expr": "sum(databricks_mock_api_response_time_count)", + "legendFormat": "total", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming requests (Total)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(databricks_mock_api_response_time_count{code!=\"200\"}[1m]))", - "hide": false, - "legendFormat": "total", - "refId": "A" - }, - { - "expr": "sum(rate(databricks_mock_api_response_time_count{code!=\"200\"}[1m])) by (type, action, code)", - "legendFormat": "{{type}} - {{action}} - {{code}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Error rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(databricks_mock_api_response_time_count{code!=\"200\"}[1m]))", + "hide": false, + "legendFormat": "total", + "refId": "A" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "expr": "sum(rate(databricks_mock_api_response_time_count{code!=\"200\"}[1m])) by (type, action, code)", + "legendFormat": "{{type}} - {{action}} - {{code}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Error rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "cps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, - "yaxes": [ - { - "format": "cps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } + ], + "yaxis": { + "align": false, + "alignLevel": null } - ], - "refresh": "5s", - "schemaVersion": 20, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Databricks Mock API", - "uid": "2BZpSTfWz", - "version": 4 - } \ No newline at end of file + } + ], + "refresh": "5s", + "schemaVersion": 20, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Databricks Mock API", + "uid": "2BZpSTfWz", + "version": 4 + } \ No newline at end of file diff --git a/go.mod b/go.mod index b0c539c..1489127 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/onsi/ginkgo v1.10.3 github.com/onsi/gomega v1.7.0 github.com/prometheus/client_golang v0.9.2 + github.com/ryotarai/prometheus-query v0.0.0-20181009101647-ea75bc6bc1b1 github.com/spf13/pflag v1.0.5 // indirect github.com/stephanos/clock v0.0.0-20161224195152-e4ec0ab5053e github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum index 1ea16c6..4a45f6e 100644 --- a/go.sum +++ b/go.sum @@ -217,6 +217,8 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nL github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/ryotarai/prometheus-query v0.0.0-20181009101647-ea75bc6bc1b1 h1:Myn0mha83ZFWB98guDCZNz1NBxhnkxnrgeAgN7yYQYs= +github.com/ryotarai/prometheus-query v0.0.0-20181009101647-ea75bc6bc1b1/go.mod h1:6Zg5OEV1ovOQcYwo2NdhVEcZjMGdnzVIMoNdgtkJBH0= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= diff --git a/portforwards.sh b/portforwards.sh index 832a5ef..7fbc0a7 100644 --- a/portforwards.sh +++ b/portforwards.sh @@ -3,6 +3,17 @@ # Find all already running kubectl port-forwards and kill them ps aux | grep [k]ubectl | awk '{print $2}' | xargs kill > /dev/null 2>&1 +echo "-------> Verify all pods up" +echo "-------> Wait for Operator to be ready" +# Ensure there are no Databricks Operator pods not in the Ready state +JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n azure-databricks-operator-system -lcontrol-plane=controller-manager get pods -o jsonpath="$JSONPATH" 2>&1 | grep -q -v "Ready=False"; do sleep 5;echo "--------> waiting for all operators to be Ready"; kubectl get pods -n azure-databricks-operator-system; done +echo "-------> Wait for MockAPI to be ready" +# Ensure there are no MockAPI pods not in the Ready state +JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n databricks-mock-api -lapp=databricks-mock-api get pods -o jsonpath="$JSONPATH" 2>&1 | grep -q -v "Ready=fALSE"; do sleep 5;echo "--------> waiting for all mocks to be Ready"; kubectl get pods -n databricks-mock-api; done +# Ensure there are no locust pods not in the Ready state +echo "-------> Wait for locust to be ready" +JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'; until kubectl -n locust get pods -lapp=locust-loadtest -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do sleep 5;echo "--------> waiting for locust to be available"; kubectl get pods -n locust; done + echo "-------> Open port-forwards" kubectl port-forward service/prom-azure-databricks-operator-grafana -n default 8080:80 & kubectl port-forward service/prom-azure-databricks-oper-prometheus -n default 9091:9090 & diff --git a/verify_load_tests.go b/verify_load_tests.go new file mode 100644 index 0000000..1e10721 --- /dev/null +++ b/verify_load_tests.go @@ -0,0 +1,70 @@ +package main + +import ( + "fmt" + "os" + "time" + + prometheus "github.com/ryotarai/prometheus-query/client" +) + +func main() { + os.Exit(99) + + client, _ := prometheus.NewClient("http://localhost:9091") + userCount := getQueryResult(client, "locust_user_count") + + for userCount < 25 { + fmt.Printf("User count: %v\n", userCount) + failRatio := getQueryResult(client, "sum(locust_fail_ratio)") + fmt.Printf("Locust failure ratio: %v\n", failRatio) + if failRatio > 0 { + onFailure("locust_fail_ratio is higher than 0") + } + + numOfFailures := getQueryResult(client, "locust_requests_num_failures{path=\"poll_run_await_completion\"}") + fmt.Printf("Number of failed locust requests: %v\n", numOfFailures) + if numOfFailures > 0 { + onFailure("locust_requests_num_failures is higher than 0") + } + + databricksFailures := getQueryResult(client, "sum(databricks_request_duration_seconds_count{object_type=\"runs\", outcome=\"failure\"})") + fmt.Printf("Number of failed databricks requests: %v\n", databricksFailures) + if databricksFailures > 0 { + onFailure("databricks_request_duration_seconds_count failure count is higher than 0") + } + + time.Sleep(10 * time.Second) + userCount = getQueryResult(client, "locust_user_count") + } + fmt.Printf("Load test finished with user count: %v\n", userCount) +} + +func getQueryResult(client *prometheus.Client, query string) float64 { + now := time.Now() + step, _ := time.ParseDuration("15s") + + resp, err := client.QueryRange(query, now, now, step) + + if err != nil { + onError(err) + } + + if len(resp.Data.Result) == 0 { + return 0 + } + + metric, err := resp.Data.Result[0].Values[0].Value() + + return metric +} + +func onFailure(err string) { + fmt.Println(err) + os.Exit(99) +} + +func onError(err error) { + fmt.Println(err) + os.Exit(1) +}