Add timeouts for load test

Makefile

@@ -309,6 +309,6 @@ run-load-testing-auto-start: set-auto-start run-load-testing
 
 set-auto-start:
 	# Args passed to locust must be in CSV format as passed in "command" section of yaml doc
-	$(eval LOCUST_ARGS=,'--no-web', '-c', '25', '-r', '0.08') 	
+	$(eval LOCUST_ARGS=,'--no-web', '-c', '25', '-r', '0.08', '--run-time', '7m') 	
 
 test-local: test-locust test-mock-api run-load-testing-auto-start

hack/verify_load_tests/main.go

@@ -10,32 +10,44 @@ import (
 
 func main() {
 	client, _ := prometheus.NewClient("http://localhost:9091")
-	userCount := getQueryResult(client, "locust_user_count")
-
-	for userCount < 25 {
-		fmt.Printf("User count: %v\n", userCount)
-		failRatio := getQueryResult(client, "sum(locust_fail_ratio)")
-		fmt.Printf("Locust failure ratio: %v\n", failRatio)
-		if failRatio > 0 {
-			onFailure("locust_fail_ratio is higher than 0")
-		}
+	timeout := time.After(7 * time.Minute)
+	tick := time.Tick(10 * time.Second)
 
-		numOfFailures := getQueryResult(client, "locust_requests_num_failures{path=\"poll_run_await_completion\"}")
-		fmt.Printf("Number of failed locust requests: %v\n", numOfFailures)
-		if numOfFailures > 0 {
-			onFailure("locust_requests_num_failures is higher than 0")
-		}
+	// Keep trying until we're timed out, got a result or got an error
+	for {
+		select {
+		// Got a timeout! fail with a timeout error
+		case <-timeout:
+			onFailure("Load test timed out")
+		// Got a tick, we should check on doSomething()
+		case <-tick:
+			userCount := getQueryResult(client, "locust_user_count")
+			if userCount >= 25 {
+				fmt.Printf("Load test finished with user count: %v\n", userCount)
+				return
+			}
 
-		databricksFailures := getQueryResult(client, "sum(databricks_request_duration_seconds_count{object_type=\"runs\", outcome=\"failure\"})")
-		fmt.Printf("Number of failed databricks requests: %v\n", databricksFailures)
-		if databricksFailures > 0 {
-			onFailure("databricks_request_duration_seconds_count failure count is higher than 0")
-		}
+			fmt.Printf("User count: %v\n", userCount)
+			failRatio := getQueryResult(client, "sum(locust_fail_ratio)")
+			fmt.Printf("Locust failure ratio: %v\n", failRatio)
+			if failRatio > 0 {
+				onFailure("locust_fail_ratio is higher than 0")
+			}
 
-		time.Sleep(10 * time.Second)
-		userCount = getQueryResult(client, "locust_user_count")
+			numOfFailures := getQueryResult(client, "locust_requests_num_failures{path=\"poll_run_await_completion\"}")
+			fmt.Printf("Number of failed locust requests: %v\n", numOfFailures)
+			if numOfFailures > 0 {
+				onFailure("locust_requests_num_failures is higher than 0")
+			}
+
+			databricksFailures := getQueryResult(client, "sum(databricks_request_duration_seconds_count{object_type=\"runs\", outcome=\"failure\"})")
+			fmt.Printf("Number of failed databricks requests: %v\n", databricksFailures)
+			if databricksFailures > 0 {
+				onFailure("databricks_request_duration_seconds_count failure count is higher than 0")
+			}
+
+		}
 	}
-	fmt.Printf("Load test finished with user count: %v\n", userCount)
 }
 
 func getQueryResult(client *prometheus.Client, query string) float64 {