Add running load tests in pipeline

Makefile

@@ -275,6 +275,8 @@ test-locust:
 	pytest
 
 port-forward:
+	@echo "$(shell tput setaf 10)$(shell tput bold)Set up port-forwarding $(shell tput sgr0)" 
+
 	./portforwards.sh
 
 create-db-mock-secret: create-namespace
@@ -297,31 +299,6 @@ deploy-cluster-for-load-testing: create-kindcluster install-prometheus create-db
 	kubectl apply -f ./config/prometheus/grafana-dashboard-mockapi-configmap.yaml
 
 run-load-testing: deploy-cluster-for-load-testing deploy-locust port-forward
+	@echo "$(shell tput setaf 10)$(shell tput bold)Verify load tests $(shell tput sgr0)" 
 
-	##
-	# python3 ./test.py
-
-	sleep 45
-	curl localhost:9090 > promstats-locust.txt
-
-
-	go get -u github.com/ryotarai/prometheus-query
-
-	prometheus-query -server http://localhost:9091 -query locust_user_count -start "now" -end "now" | jq .[0].values[0].value	
-
-	# LOCUST_ARGS?="'--noweb', '-c', '25', '-r', '0.03'"
-
-	# python3 ./test.py ./promstats-lucust.txt
-
-	# Check stats
-	# while "curl localhost:9090" prom metics "locust_user_count" < 25... wait
-
-	# get prommetrics and extract stats
-	# check against thresholds 
-
-	# pass or fail!
-
-	# prometheus-query -server http://localhost:9091 -query locust_user_count -start "now" -end "now" | jq .[0].values[0].value
-	# # LOCUST_ARGS?=,'--no-web', '-c', '25', '-r', '0.08
-
-
+	go run verify_load_tests.go

azure-pipelines.yaml

@@ -83,6 +83,10 @@ jobs:
       failOnStandardError: false
       inlineScript: az acr build --registry $(ACR_NAME) --image $(IMAGE_NAME) .
 
+  - script: make run-load-testing
+    condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master'))
+    displayName: 'Run load tests'
+
   - script: |
       ./bin/kustomize build config/default > $(Build.ArtifactStagingDirectory)/setup.yaml
       set -x

config/prometheus/grafana-dashboard-load-test-configmap.yaml

@@ -23,7 +23,7 @@ data:
       "editable": true,
       "gnetId": null,
       "graphTooltip": 1,
-      "iteration": 1576783033807,
+      "iteration": 1580822557510,
       "links": [],
       "panels": [
         {
@@ -2138,7 +2138,7 @@ data:
             "x": 0,
             "y": 54
           },
-          "id": 50,
+          "id": 53,
           "legend": {
             "avg": false,
             "current": false,
@@ -2226,7 +2226,7 @@ data:
           "dashLength": 10,
           "dashes": false,
           "datasource": null,
-          "description": "Run duration (as returned by Databricks API)",
+          "description": "Run duration (from API) and the time from Run start time (as per API) to time the operator reconciles the terminated state and the run duration (from API)",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -2235,7 +2235,7 @@ data:
             "x": 16,
             "y": 60
           },
-          "id": 46,
+          "id": 48,
           "legend": {
             "avg": false,
             "current": false,
@@ -2261,16 +2261,21 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(databricks_run_duration_seconds_sum[1m])/rate(databricks_run_duration_seconds_count[1m])",
-              "legendFormat": "mean_duration-{{life_cycle_state}}",
+              "expr": "rate(databricks_run_time_to_detect_finished_seconds_sum[1m])/rate(databricks_run_time_to_detect_finished_seconds_count[1m])",
+              "legendFormat": "time to operator complete - {{life_cycle_state}}",
               "refId": "A"
+            },
+            {
+              "expr": "rate(databricks_run_duration_seconds_sum[1m])/rate(databricks_run_duration_seconds_count[1m])",
+              "legendFormat": "run duration - {{life_cycle_state}}",
+              "refId": "B"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Run Duration",
+          "title": "Run duration",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -2288,7 +2293,7 @@ data:
             {
               "decimals": 2,
               "format": "s",
-              "label": "",
+              "label": null,
               "logBase": 1,
               "max": null,
               "min": null,
@@ -2314,7 +2319,7 @@ data:
           "dashLength": 10,
           "dashes": false,
           "datasource": null,
-          "description": "Time from Run start time (as per API) to time the operator reconciles the terminated state",
+          "description": "Latency in operator.\nDefined as the difference between the time from Run start time (as per API) to time the operator reconciles the terminated state and the run duration (from API)",
           "fill": 1,
           "fillGradient": 0,
           "gridPos": {
@@ -2323,7 +2328,7 @@ data:
             "x": 16,
             "y": 68
           },
-          "id": 48,
+          "id": 52,
           "legend": {
             "avg": false,
             "current": false,
@@ -2349,16 +2354,16 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "rate(databricks_run_time_to_detect_finished_seconds_sum[1m])/rate(databricks_run_time_to_detect_finished_seconds_count[1m])",
-              "legendFormat": "mean_time-{{life_cycle_state}}",
+              "expr": "rate(databricks_run_time_to_detect_finished_seconds_sum[1m])/rate(databricks_run_time_to_detect_finished_seconds_count[1m]) - rate(databricks_run_duration_seconds_sum[1m])/rate(databricks_run_duration_seconds_count[1m])",
+              "legendFormat": "operator latency - {{life_cycle_state}}",
               "refId": "A"
             }
           ],
           "thresholds": [],
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "Time to Run Completion",
+          "title": "Operator run latency",
           "tooltip": {
             "shared": true,
             "sort": 0,