system-tests: ran-du: add tuned restarts check to stability tests

tests/system-tests/internal/stability/stability.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -130,6 +131,49 @@ func SavePolicyStatus(apiClient *clients.Settings, clusterName string, outputFil
 	return nil
 }
 
+// SaveTunedRestarts stores the status of tuned restarts in the outputFile.
+func SaveTunedRestarts(apiClient *clients.Settings, outputFile string) error {
+	file, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	// Get the tuned pod by name
+	tunedPods, err := pod.ListByNamePattern(apiClient, "tuned", "openshift-cluster-node-tuning-operator")
+	if err != nil {
+		return err
+	}
+
+	tunedRestarts := make(map[string]string)
+
+	for _, tunedPod := range tunedPods {
+		tunedLog, err := tunedPod.GetFullLog("tuned")
+		if err != nil {
+			return err
+		}
+
+		pattern := "static tuning from profile .* applied"
+
+		regex, err := regexp.Compile(pattern)
+		if err != nil {
+			return err
+		}
+
+		tunedApplyCount := len(regex.FindAllString(tunedLog, -1))
+		tunedRestarts["tuned_restarts"] = fmt.Sprintf("%d", tunedApplyCount)
+	}
+
+	entry := buildOutputLine(tunedRestarts)
+
+	_, err = file.WriteString(entry)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // VerifyStabilityStatusChange checks if there has been a change between in
 // the column of the stability output file.
 func VerifyStabilityStatusChange(filePath string) (bool, error) {

tests/system-tests/ran-du/tests/stability-no-workload.go

@@ -41,6 +41,7 @@ var _ = Describe(
 			outputDir := RanDuTestConfig.StabilityOutputPath
 			policiesOutputFile := fmt.Sprintf("%s/stability_no_workload_policies.log", outputDir)
 			ptpOutputFile := fmt.Sprintf("%s/stability_no_workload_ptp.log", outputDir)
+			tunedRestartsOutputFile := fmt.Sprintf("%s/stability_tuned_restarts.log", outputDir)
 			namespaces := []string{"openshift-etcd", "openshift-apiserver"}
 
 			totalDuration := time.Duration(RanDuTestConfig.StabilityNoWorkloadDurMins) * time.Minute
@@ -71,6 +72,11 @@ var _ = Describe(
 
 				}
 
+				err = stability.SaveTunedRestarts(APIClient, tunedRestartsOutputFile)
+				if err != nil {
+					fmt.Printf("Error, could not save tuned restarts")
+				}
+
 				time.Sleep(interval)
 			}
 
@@ -105,6 +111,13 @@ var _ = Describe(
 				}
 			}
 
+			// Verify tuned restarts
+			By("Check tuneds restarts")
+			_, err = stability.VerifyStabilityStatusChange(tunedRestartsOutputFile)
+			if err != nil {
+				stabilityErrors = append(stabilityErrors, err.Error())
+			}
+
 			By("Check if there been any error")
 			if len(stabilityErrors) > 0 {
 				Expect(stabilityErrors).ToNot(HaveOccurred(), "One or more errors in stability tests:%s", stabilityErrors)

tests/system-tests/ran-du/tests/stability-workload.go

@@ -62,6 +62,7 @@ var _ = Describe(
 			outputDir := RanDuTestConfig.StabilityOutputPath
 			policiesOutputFile := fmt.Sprintf("%s/stability_workload_policies.log", outputDir)
 			ptpOutputFile := fmt.Sprintf("%s/stability_workload_ptp.log", outputDir)
+			tunedRestartsOutputFile := fmt.Sprintf("%s/stability_tuned_restarts.log", outputDir)
 			namespaces := []string{"openshift-etcd", "openshift-apiserver"}
 
 			totalDuration := time.Duration(RanDuTestConfig.StabilityWorkloadDurMins) * time.Minute
@@ -90,6 +91,11 @@ var _ = Describe(
 					}
 				}
 
+				err = stability.SaveTunedRestarts(APIClient, tunedRestartsOutputFile)
+				if err != nil {
+					fmt.Printf("Error, could not save tuned restarts")
+				}
+
 				time.Sleep(interval)
 			}
 
@@ -122,6 +128,13 @@ var _ = Describe(
 				}
 			}
 
+			// Verify tuned restarts
+			By("Check tuneds restarts")
+			_, err = stability.VerifyStabilityStatusChange(tunedRestartsOutputFile)
+			if err != nil {
+				stabilityErrors = append(stabilityErrors, err.Error())
+			}
+
 			By("Check if there been any error")
 			if len(stabilityErrors) > 0 {
 				Expect(stabilityErrors).ToNot(HaveOccurred(), "One or more errors in stability tests: %s", stabilityErrors)