Skip to content

Commit 8d343a8

Browse files
vipul-21vipul-21
committed
refactor: [CI-CNI] Removing the predefined interfaces from validate package
1 parent 2dff809 commit 8d343a8

File tree

5 files changed

+159
-199
lines changed

5 files changed

+159
-199
lines changed

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -741,7 +741,7 @@ test-integration: ## run all integration tests.
741741
go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...
742742

743743
test-validate-state:
744-
cd test/integration/load && go test -count 1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE) -os=$(OS)
744+
cd test/integration/load && go test -mod=readonly -count=1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE) -os=$(OS)
745745
cd ../../..
746746

747747
test-cyclonus: ## run the cyclonus test for npm.

‎test/integration/load/load_test.go

+21-16
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,10 @@ todo: consider adding the following scenarios
5252
- [x] Test the CNS Local cache.
5353
- [x] Test the Cilium state file.
5454
- [x] Test the Node restart.
55-
- [ ] Test based on operating system.
56-
- [ ] Test the HNS state file.
57-
- [ ] Parameterize the os, cni and number of iterations.
58-
- [ ] Add deployment yaml for windows.
55+
- [x] Test based on operating system.
56+
- [x] Test the HNS state file.
57+
- [x] Parameterize the os, cni and number of iterations.
58+
- [x] Add deployment yaml for windows.
5959
*/
6060
func TestLoad(t *testing.T) {
6161
clientset, err := k8sutils.MustGetClientset()
@@ -131,21 +131,26 @@ func TestValidateState(t *testing.T) {
131131
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
132132
defer cancel()
133133

134-
t.Log("Validating the state file")
135-
validatorClient := validate.GetValidatorClient(*osType)
136-
validator := validatorClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase)
134+
if *osType == "windows" {
135+
t.Log("Validating the windows state file")
136+
windowsClient := &validate.WindowsClient{}
137+
windowsClient = windowsClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase)
137138

138-
err = validator.ValidateStateFile()
139-
if err != nil {
140-
t.Fatal(err)
141-
}
139+
err = windowsClient.Validate()
140+
if err != nil {
141+
t.Fatal(err)
142+
}
143+
} else if *osType == "linux" {
144+
t.Log("Validating the linux state file and restart network scenario")
145+
linuxClient := &validate.LinuxClient{}
146+
linuxClient = linuxClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase)
142147

143-
//We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706
144-
t.Log("Validating the restart network scenario")
145-
err = validator.ValidateRestartNetwork()
146-
if err != nil {
147-
t.Fatal(err)
148+
err = linuxClient.Validate()
149+
if err != nil {
150+
t.Fatal(err)
151+
}
148152
}
153+
149154
}
150155

151156
// TestScaleDeployment scales the deployment up/down based on the replicas passed.

‎test/validate/client.go

+83-17
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,105 @@ package validate
22

33
import (
44
"context"
5+
"log"
56

7+
k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils"
8+
"github.com/pkg/errors"
69
"k8s.io/client-go/kubernetes"
710
"k8s.io/client-go/rest"
811
)
912

10-
type Validator struct {
13+
type validator struct {
1114
ctx context.Context
1215
clientset *kubernetes.Clientset
1316
config *rest.Config
17+
checks []check
1418
namespace string
1519
cni string
1620
restartCase bool
1721
}
1822

19-
// Todo: Add the validation for the data path function for the linux/windows client.
20-
type IValidator interface {
21-
ValidateStateFile() error
22-
ValidateRestartNetwork() error
23-
// ValidateDataPath() error
23+
type check struct {
24+
name string
25+
stateFileIps func([]byte) (map[string]string, error)
26+
podLabelSelector string
27+
podNamespace string
28+
cmd []string
2429
}
2530

26-
type validatorClient interface {
27-
CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator
31+
func (v *validator) ValidateStateFile() error {
32+
for _, check := range v.checks {
33+
err := v.validateIPs(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector)
34+
if err != nil {
35+
return err
36+
}
37+
}
38+
return nil
2839
}
2940

30-
// Func to get the type of validator client based on the Operating system.
31-
func GetValidatorClient(os string) validatorClient {
32-
switch os {
33-
case "linux":
34-
return &LinuxClient{}
35-
case "windows":
36-
return &WindowsClient{}
37-
default:
38-
return nil
41+
func (v *validator) ValidateRestartNetwork() error {
42+
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
43+
if err != nil {
44+
return errors.Wrapf(err, "failed to get node list")
45+
}
46+
47+
for index := range nodes.Items {
48+
// get the privileged pod
49+
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name)
50+
if err != nil {
51+
return errors.Wrapf(err, "failed to get privileged pod")
52+
}
53+
54+
privelegedPod := pod.Items[0]
55+
// exec into the pod to get the state file
56+
_, err = k8sutils.ExecCmdOnPod(v.ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config)
57+
if err != nil {
58+
return errors.Wrapf(err, "failed to exec into privileged pod")
59+
}
60+
err = k8sutils.WaitForPodsRunning(v.ctx, v.clientset, "", "")
61+
if err != nil {
62+
return errors.Wrapf(err, "failed to wait for pods running")
63+
}
64+
}
65+
return nil
66+
}
67+
68+
func (v *validator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error {
69+
log.Printf("Validating %s state file", checkType)
70+
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
71+
if err != nil {
72+
return errors.Wrapf(err, "failed to get node list")
73+
}
74+
75+
for index := range nodes.Items {
76+
// get the privileged pod
77+
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name)
78+
if err != nil {
79+
return errors.Wrapf(err, "failed to get privileged pod")
80+
}
81+
podName := pod.Items[0].Name
82+
// exec into the pod to get the state file
83+
result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, namespace, podName, cmd, v.config)
84+
if err != nil {
85+
return errors.Wrapf(err, "failed to exec into privileged pod")
86+
}
87+
filePodIps, err := stateFileIps(result)
88+
if err != nil {
89+
return errors.Wrapf(err, "failed to get pod ips from state file")
90+
}
91+
if len(filePodIps) == 0 && v.restartCase {
92+
log.Printf("No pods found on node %s", nodes.Items[index].Name)
93+
continue
94+
}
95+
// get the pod ips
96+
podIps := getPodIPsWithoutNodeIP(v.ctx, v.clientset, nodes.Items[index])
97+
98+
check := compareIPs(filePodIps, podIps)
99+
100+
if !check {
101+
return errors.Wrapf(errors.New("State file validation failed"), "for %s on node %s", checkType, nodes.Items[index].Name)
102+
}
39103
}
104+
log.Printf("State file validation for %s passed", checkType)
105+
return nil
40106
}

‎test/validate/linux_validate.go

+31-90
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,20 @@ var (
3131

3232
type stateFileIpsFunc func([]byte) (map[string]string, error)
3333

34-
type LinuxClient struct{}
34+
type linuxValidator interface {
35+
ValidateStateFile() error
36+
ValidateRestartNetwork() error
37+
}
3538

36-
type LinuxValidator struct {
37-
Validator
39+
var linuxChecksMap = map[string][]check{
40+
"cilium": {
41+
{"cns", cnsStateFileIps, cnsLabelSelector, privilegedNamespace, cnsStateFileCmd},
42+
{"cilium", ciliumStateFileIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd},
43+
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd},
44+
},
45+
"cniv2": {
46+
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd},
47+
},
3848
}
3949

4050
type CnsState struct {
@@ -62,7 +72,11 @@ type Address struct {
6272
Addr string `json:"ipv4"`
6373
}
6474

65-
func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator {
75+
type LinuxClient struct {
76+
validator linuxValidator
77+
}
78+
79+
func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) *LinuxClient {
6680
// deploy privileged pod
6781
privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedDaemonSetPath)
6882
if err != nil {
@@ -77,64 +91,31 @@ func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Cli
7791
if err != nil {
7892
panic(err)
7993
}
80-
return &LinuxValidator{
81-
Validator: Validator{
94+
95+
return &LinuxClient{
96+
validator: &validator{
8297
ctx: ctx,
8398
clientset: clienset,
8499
config: config,
85100
namespace: namespace,
86101
cni: cni,
87102
restartCase: restartCase,
103+
checks: linuxChecksMap[cni],
88104
},
89105
}
90106
}
91107

92-
// Todo: Based on cni version validate different state files
93-
func (v *LinuxValidator) ValidateStateFile() error {
94-
checkSet := make(map[string][]check) // key is cni type, value is a list of check
95-
// TODO: add cniv1 when adding Linux related test cases
96-
checkSet["cilium"] = []check{
97-
{"cns", cnsStateFileIps, cnsLabelSelector, privilegedNamespace, cnsStateFileCmd},
98-
{"cilium", ciliumStateFileIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd},
99-
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd},
100-
}
101-
102-
checkSet["cniv2"] = []check{
103-
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd},
104-
}
105-
106-
for _, check := range checkSet[v.cni] {
107-
err := v.validateIPs(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector)
108-
if err != nil {
109-
return err
110-
}
111-
}
112-
return nil
113-
}
114-
115-
func (v *LinuxValidator) ValidateRestartNetwork() error {
116-
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
108+
func (l *LinuxClient) Validate() error {
109+
log.Printf("Validating State File")
110+
err := l.validator.ValidateStateFile()
117111
if err != nil {
118-
return errors.Wrapf(err, "failed to get node list")
112+
return errors.Wrapf(err, "failed to validate state file")
119113
}
120-
121-
for index := range nodes.Items {
122-
// get the privileged pod
123-
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name)
124-
if err != nil {
125-
return errors.Wrapf(err, "failed to get privileged pod")
126-
}
127-
128-
privelegedPod := pod.Items[0]
129-
// exec into the pod to get the state file
130-
_, err = k8sutils.ExecCmdOnPod(v.ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config)
131-
if err != nil {
132-
return errors.Wrapf(err, "failed to exec into privileged pod")
133-
}
134-
err = k8sutils.WaitForPodsRunning(v.ctx, v.clientset, "", "")
135-
if err != nil {
136-
return errors.Wrapf(err, "failed to wait for pods running")
137-
}
114+
// We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706
115+
log.Printf("Validating the restart network scenario")
116+
err = l.validator.ValidateRestartNetwork()
117+
if err != nil {
118+
return errors.Wrapf(err, "failed to validate restart network scenario")
138119
}
139120
return nil
140121
}
@@ -190,43 +171,3 @@ func cnsCacheStateFileIps(result []byte) (map[string]string, error) {
190171
}
191172
return cnsPodIps, nil
192173
}
193-
194-
func (v *LinuxValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error {
195-
log.Printf("Validating %s state file", checkType)
196-
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
197-
if err != nil {
198-
return errors.Wrapf(err, "failed to get node list")
199-
}
200-
201-
for index := range nodes.Items {
202-
// get the privileged pod
203-
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name)
204-
if err != nil {
205-
return errors.Wrapf(err, "failed to get privileged pod")
206-
}
207-
podName := pod.Items[0].Name
208-
// exec into the pod to get the state file
209-
result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, namespace, podName, cmd, v.config)
210-
if err != nil {
211-
return errors.Wrapf(err, "failed to exec into privileged pod")
212-
}
213-
filePodIps, err := stateFileIps(result)
214-
if err != nil {
215-
return errors.Wrapf(err, "failed to get pod ips from state file")
216-
}
217-
if len(filePodIps) == 0 && v.restartCase {
218-
log.Printf("No pods found on node %s", nodes.Items[index].Name)
219-
continue
220-
}
221-
// get the pod ips
222-
podIps := getPodIPsWithoutNodeIP(v.ctx, v.clientset, nodes.Items[index])
223-
224-
check := compareIPs(filePodIps, podIps)
225-
226-
if !check {
227-
return errors.Wrapf(errors.New("State file validation failed"), "for %s on node %s", checkType, nodes.Items[index].Name)
228-
}
229-
}
230-
log.Printf("State file validation for %s passed", checkType)
231-
return nil
232-
}

0 commit comments

Comments
 (0)