Skip to content

Commit 4ef0ff2

Browse files
vipul-21vipul-21
committed
refactor: [CI-CNI] Removing the predefined interfaces from validate package
1 parent 672cce9 commit 4ef0ff2

File tree

6 files changed

+157
-194
lines changed

6 files changed

+157
-194
lines changed

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -737,7 +737,7 @@ test-integration: ## run all integration tests.
737737
go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...
738738

739739
test-validate-state:
740-
cd test/integration/load && go test -count 1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE) -os=$(OS)
740+
cd test/integration/load && go test -mod=readonly -count=1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE) -os=$(OS)
741741
cd ../../..
742742

743743
test-cyclonus: ## run the cyclonus test for npm.

‎test/integration/load/load_test.go

+23-16
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,10 @@ todo: consider adding the following scenarios
5252
- [x] Test the CNS Local cache.
5353
- [x] Test the Cilium state file.
5454
- [x] Test the Node restart.
55-
- [ ] Test based on operating system.
56-
- [ ] Test the HNS state file.
57-
- [ ] Parameterize the os, cni and number of iterations.
58-
- [ ] Add deployment yaml for windows.
55+
- [x] Test based on operating system.
56+
- [x] Test the HNS state file.
57+
- [x] Parameterize the os, cni and number of iterations.
58+
- [x] Add deployment yaml for windows.
5959
*/
6060
func TestLoad(t *testing.T) {
6161
clientset, err := k8sutils.MustGetClientset()
@@ -131,21 +131,28 @@ func TestValidateState(t *testing.T) {
131131
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
132132
defer cancel()
133133

134-
t.Log("Validating the state file")
135-
validatorClient := validate.GetValidatorClient(*osType)
136-
validator := validatorClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase)
134+
if *osType == "windows" {
135+
t.Log("Validating the windows state file")
136+
windowsClient := &validate.WindowsClient{}
137+
windowsClient = windowsClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase)
137138

138-
err = validator.ValidateStateFile()
139-
if err != nil {
140-
t.Fatal(err)
141-
}
139+
err = windowsClient.Validate()
140+
if err != nil {
141+
t.Fatal(err)
142+
}
143+
t.Log("State file validated successfully")
144+
} else if *osType == "linux" {
145+
t.Log("Validating the linux state file and restart network scenario")
146+
linuxClient := &validate.LinuxClient{}
147+
linuxClient = linuxClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase)
142148

143-
//We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706
144-
t.Log("Validating the restart network scenario")
145-
err = validator.ValidateRestartNetwork()
146-
if err != nil {
147-
t.Fatal(err)
149+
err = linuxClient.Validate()
150+
if err != nil {
151+
t.Fatal(err)
152+
}
148153
}
154+
155+
t.Log("State file validated successfully")
149156
}
150157

151158
// TestScaleDeployment scales the deployment up/down based on the replicas passed.

‎test/integration/manifests/noop-deployment-windows.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ spec:
1616
spec:
1717
containers:
1818
- name: noop
19-
image: mcr.microsoft.com/windows/nanoserver:ltsc2022
19+
image: mcr.microsoft.com/oss/kubernetes/pause:3.6-windows-ltsc2022-amd64
2020
ports:
2121
- containerPort: 80
2222
nodeSelector:

‎test/validate/client.go

+83-17
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,105 @@ package validate
22

33
import (
44
"context"
5+
"log"
56

7+
k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils"
8+
"github.com/pkg/errors"
69
"k8s.io/client-go/kubernetes"
710
"k8s.io/client-go/rest"
811
)
912

10-
type Validator struct {
13+
type validator struct {
1114
ctx context.Context
1215
clientset *kubernetes.Clientset
1316
config *rest.Config
17+
checks []check
1418
namespace string
1519
cni string
1620
restartCase bool
1721
}
1822

19-
// Todo: Add the validation for the data path function for the linux/windows client.
20-
type IValidator interface {
21-
ValidateStateFile() error
22-
ValidateRestartNetwork() error
23-
// ValidateDataPath() error
23+
type check struct {
24+
name string
25+
stateFileIps func([]byte) (map[string]string, error)
26+
podLabelSelector string
27+
podNamespace string
28+
cmd []string
2429
}
2530

26-
type validatorClient interface {
27-
CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator
31+
func (v *validator) ValidateStateFile() error {
32+
for _, check := range v.checks {
33+
err := v.validate(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector)
34+
if err != nil {
35+
return err
36+
}
37+
}
38+
return nil
2839
}
2940

30-
// Func to get the type of validator client based on the Operating system.
31-
func GetValidatorClient(os string) validatorClient {
32-
switch os {
33-
case "linux":
34-
return &LinuxClient{}
35-
case "windows":
36-
return &WindowsClient{}
37-
default:
38-
return nil
41+
func (v *validator) ValidateRestartNetwork() error {
42+
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
43+
if err != nil {
44+
return errors.Wrapf(err, "failed to get node list")
45+
}
46+
47+
for index := range nodes.Items {
48+
// get the privileged pod
49+
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name)
50+
if err != nil {
51+
return errors.Wrapf(err, "failed to get privileged pod")
52+
}
53+
54+
privelegedPod := pod.Items[0]
55+
// exec into the pod to get the state file
56+
_, err = k8sutils.ExecCmdOnPod(v.ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config)
57+
if err != nil {
58+
return errors.Wrapf(err, "failed to exec into privileged pod")
59+
}
60+
err = k8sutils.WaitForPodsRunning(v.ctx, v.clientset, "", "")
61+
if err != nil {
62+
return errors.Wrapf(err, "failed to wait for pods running")
63+
}
64+
}
65+
return nil
66+
}
67+
68+
func (v *validator) validate(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error {
69+
log.Printf("Validating %s state file", checkType)
70+
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
71+
if err != nil {
72+
return errors.Wrapf(err, "failed to get node list")
73+
}
74+
75+
for index := range nodes.Items {
76+
// get the privileged pod
77+
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name)
78+
if err != nil {
79+
return errors.Wrapf(err, "failed to get privileged pod")
80+
}
81+
podName := pod.Items[0].Name
82+
// exec into the pod to get the state file
83+
result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, namespace, podName, cmd, v.config)
84+
if err != nil {
85+
return errors.Wrapf(err, "failed to exec into privileged pod")
86+
}
87+
filePodIps, err := stateFileIps(result)
88+
if err != nil {
89+
return errors.Wrapf(err, "failed to get pod ips from state file")
90+
}
91+
if len(filePodIps) == 0 && v.restartCase {
92+
log.Printf("No pods found on node %s", nodes.Items[index].Name)
93+
continue
94+
}
95+
// get the pod ips
96+
podIps := getPodIPsWithoutNodeIP(v.ctx, v.clientset, nodes.Items[index])
97+
98+
check := compareIPs(filePodIps, podIps)
99+
100+
if !check {
101+
return errors.Wrapf(errors.New("State file validation failed"), "for %s on node %s", checkType, nodes.Items[index].Name)
102+
}
39103
}
104+
log.Printf("State file validation for %s passed", checkType)
105+
return nil
40106
}

‎test/validate/linux_validate.go

+28-90
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,17 @@ var (
3131

3232
type stateFileIpsFunc func([]byte) (map[string]string, error)
3333

34-
type LinuxClient struct{}
34+
type linuxValidator interface {
35+
ValidateStateFile() error
36+
ValidateRestartNetwork() error
37+
}
3538

36-
type LinuxValidator struct {
37-
Validator
39+
var linuxChecksMap = map[string][]check{
40+
"cilium": {
41+
{"cns", cnsStateFileIps, cnsLabelSelector, privilegedNamespace, cnsStateFileCmd},
42+
{"cilium", ciliumStateFileIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd},
43+
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd},
44+
},
3845
}
3946

4047
type CnsState struct {
@@ -62,7 +69,11 @@ type Address struct {
6269
Addr string `json:"ipv4"`
6370
}
6471

65-
func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator {
72+
type LinuxClient struct {
73+
validator linuxValidator
74+
}
75+
76+
func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) *LinuxClient {
6677
// deploy privileged pod
6778
privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedDaemonSetPath)
6879
if err != nil {
@@ -77,64 +88,31 @@ func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Cli
7788
if err != nil {
7889
panic(err)
7990
}
80-
return &LinuxValidator{
81-
Validator: Validator{
91+
92+
return &LinuxClient{
93+
validator: &validator{
8294
ctx: ctx,
8395
clientset: clienset,
8496
config: config,
8597
namespace: namespace,
8698
cni: cni,
8799
restartCase: restartCase,
100+
checks: linuxChecksMap[cni],
88101
},
89102
}
90103
}
91104

92-
// Todo: Based on cni version validate different state files
93-
func (v *LinuxValidator) ValidateStateFile() error {
94-
checks := []struct {
95-
name string
96-
stateFileIps func([]byte) (map[string]string, error)
97-
podLabelSelector string
98-
podNamespace string
99-
cmd []string
100-
}{
101-
{"cns", cnsStateFileIps, cnsLabelSelector, privilegedNamespace, cnsStateFileCmd},
102-
{"cilium", ciliumStateFileIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd},
103-
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd},
104-
}
105-
106-
for _, check := range checks {
107-
err := v.validate(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector)
108-
if err != nil {
109-
return err
110-
}
111-
}
112-
return nil
113-
}
114-
115-
func (v *LinuxValidator) ValidateRestartNetwork() error {
116-
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
105+
func (l *LinuxClient) Validate() error {
106+
log.Printf("Validating State File")
107+
err := l.validator.ValidateStateFile()
117108
if err != nil {
118-
return errors.Wrapf(err, "failed to get node list")
109+
return err
119110
}
120-
121-
for index := range nodes.Items {
122-
// get the privileged pod
123-
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name)
124-
if err != nil {
125-
return errors.Wrapf(err, "failed to get privileged pod")
126-
}
127-
128-
privelegedPod := pod.Items[0]
129-
// exec into the pod to get the state file
130-
_, err = k8sutils.ExecCmdOnPod(v.ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config)
131-
if err != nil {
132-
return errors.Wrapf(err, "failed to exec into privileged pod")
133-
}
134-
err = k8sutils.WaitForPodsRunning(v.ctx, v.clientset, "", "")
135-
if err != nil {
136-
return errors.Wrapf(err, "failed to wait for pods running")
137-
}
111+
//We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706
112+
log.Printf("Validating the restart network scenario")
113+
err = l.validator.ValidateRestartNetwork()
114+
if err != nil {
115+
return err
138116
}
139117
return nil
140118
}
@@ -190,43 +168,3 @@ func cnsCacheStateFileIps(result []byte) (map[string]string, error) {
190168
}
191169
return cnsPodIps, nil
192170
}
193-
194-
func (v *LinuxValidator) validate(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error {
195-
log.Printf("Validating %s state file", checkType)
196-
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
197-
if err != nil {
198-
return errors.Wrapf(err, "failed to get node list")
199-
}
200-
201-
for index := range nodes.Items {
202-
// get the privileged pod
203-
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name)
204-
if err != nil {
205-
return errors.Wrapf(err, "failed to get privileged pod")
206-
}
207-
podName := pod.Items[0].Name
208-
// exec into the pod to get the state file
209-
result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, namespace, podName, cmd, v.config)
210-
if err != nil {
211-
return errors.Wrapf(err, "failed to exec into privileged pod")
212-
}
213-
filePodIps, err := stateFileIps(result)
214-
if err != nil {
215-
return errors.Wrapf(err, "failed to get pod ips from state file")
216-
}
217-
if len(filePodIps) == 0 && v.restartCase {
218-
log.Printf("No pods found on node %s", nodes.Items[index].Name)
219-
continue
220-
}
221-
// get the pod ips
222-
podIps := getPodIPsWithoutNodeIP(v.ctx, v.clientset, nodes.Items[index])
223-
224-
check := compareIPs(filePodIps, podIps)
225-
226-
if !check {
227-
return errors.Wrapf(errors.New("State file validation failed"), "for %s on node %s", checkType, nodes.Items[index].Name)
228-
}
229-
}
230-
log.Printf("State file validation for %s passed", checkType)
231-
return nil
232-
}

0 commit comments

Comments
 (0)