Skip to content

Commit 390977d

Browse files
authored
ci:[CNI] Add windows CNIv1 datapath test (Azure#2016)
* ci: Transfer files * test: Working Datapath Test * test: apierror Tests * style: Datapath Package * test: Deployment timing * fix: Error check * fix: Lint
1 parent d4df708 commit 390977d

File tree

7 files changed

+425
-1
lines changed

7 files changed

+425
-1
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
//go:build connection
2+
3+
package connection
4+
5+
import (
6+
"context"
7+
"flag"
8+
"fmt"
9+
"testing"
10+
11+
"github.com/Azure/azure-container-networking/test/internal/datapath"
12+
"github.com/Azure/azure-container-networking/test/internal/k8sutils"
13+
"github.com/pkg/errors"
14+
"github.com/stretchr/testify/require"
15+
apiv1 "k8s.io/api/core/v1"
16+
apierrors "k8s.io/apimachinery/pkg/api/errors"
17+
)
18+
19+
const (
20+
WindowsDeployYamlPath = "../manifests/datapath/windows-deployment.yaml"
21+
podLabelKey = "app"
22+
podCount = 2
23+
nodepoolKey = "agentpool"
24+
)
25+
26+
var (
27+
podPrefix = flag.String("podName", "datapod", "Prefix for test pods")
28+
podNamespace = flag.String("namespace", "datapath-win", "Namespace for test pods")
29+
nodepoolSelector = flag.String("nodepoolSelector", "npwin", "Provides nodepool as a Node-Selector for pods")
30+
)
31+
32+
/*
33+
This test assumes that you have the current credentials loaded in your default kubeconfig for a
34+
k8s cluster with a windows nodepool consisting of at least 2 windows nodes.
35+
*** The expected nodepool name is npwin, if the nodepool has a diferent name ensure that you change nodepoolSelector with:
36+
-nodepoolSelector="yournodepoolname"
37+
38+
To run the test use one of the following commands:
39+
go test -count=1 test/integration/datapath/datapath_win_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection
40+
or
41+
go test -count=1 test/integration/datapath/datapath_win_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -podName=acnpod -nodepoolSelector=npwina -tags=connection
42+
43+
44+
This test checks pod to pod, pod to node, and pod to internet for datapath connectivity.
45+
46+
Timeout context is controled by the -timeout flag.
47+
48+
*/
49+
50+
func TestDatapathWin(t *testing.T) {
51+
ctx := context.Background()
52+
53+
t.Log("Create Clientset")
54+
clientset, err := k8sutils.MustGetClientset()
55+
if err != nil {
56+
require.NoError(t, err, "could not get k8s clientset: %v", err)
57+
}
58+
t.Log("Get REST config")
59+
restConfig := k8sutils.MustGetRestConfig(t)
60+
61+
t.Log("Create Label Selectors")
62+
podLabelSelector := fmt.Sprintf("%s=%s", podLabelKey, *podPrefix)
63+
nodeLabelSelector := fmt.Sprintf("%s=%s", nodepoolKey, *nodepoolSelector)
64+
65+
t.Log("Get Nodes")
66+
nodes, err := k8sutils.GetNodeListByLabelSelector(ctx, clientset, nodeLabelSelector)
67+
if err != nil {
68+
require.NoError(t, err, "could not get k8s node list: %v", err)
69+
}
70+
71+
// Test Namespace
72+
t.Log("Create Namespace")
73+
err = k8sutils.MustCreateNamespace(ctx, clientset, *podNamespace)
74+
createPodFlag := !(apierrors.IsAlreadyExists(err))
75+
76+
if createPodFlag {
77+
t.Log("Creating Windows pods through deployment")
78+
deployment, err := k8sutils.MustParseDeployment(WindowsDeployYamlPath)
79+
if err != nil {
80+
require.NoError(t, err)
81+
}
82+
83+
// Fields for overwritting existing deployment yaml.
84+
// Defaults from flags will not change anything
85+
deployment.Spec.Selector.MatchLabels[podLabelKey] = *podPrefix
86+
deployment.Spec.Template.ObjectMeta.Labels[podLabelKey] = *podPrefix
87+
deployment.Spec.Template.Spec.NodeSelector[nodepoolKey] = *nodepoolSelector
88+
deployment.Name = *podPrefix
89+
deployment.Namespace = *podNamespace
90+
91+
deploymentsClient := clientset.AppsV1().Deployments(*podNamespace)
92+
err = k8sutils.MustCreateDeployment(ctx, deploymentsClient, deployment)
93+
if err != nil {
94+
require.NoError(t, err)
95+
}
96+
97+
t.Log("Waiting for pods to be running state")
98+
err = k8sutils.WaitForPodsRunning(ctx, clientset, *podNamespace, podLabelSelector)
99+
if err != nil {
100+
require.NoError(t, err)
101+
}
102+
t.Log("Successfully created customer windows pods")
103+
} else {
104+
// Checks namespace already exists from previous attempt
105+
t.Log("Namespace already exists")
106+
107+
t.Log("Checking for pods to be running state")
108+
err = k8sutils.WaitForPodsRunning(ctx, clientset, *podNamespace, podLabelSelector)
109+
if err != nil {
110+
require.NoError(t, err)
111+
}
112+
}
113+
t.Log("Checking Windows test environment ")
114+
for _, node := range nodes.Items {
115+
116+
pods, err := k8sutils.GetPodsByNode(ctx, clientset, *podNamespace, podLabelSelector, node.Name)
117+
if err != nil {
118+
require.NoError(t, err, "could not get k8s clientset: %v", err)
119+
}
120+
if len(pods.Items) <= 1 {
121+
t.Logf("%s", node.Name)
122+
require.NoError(t, errors.New("Less than 2 pods on node"))
123+
}
124+
}
125+
t.Log("Windows test environment ready")
126+
127+
t.Run("Windows ping tests pod -> node", func(t *testing.T) {
128+
// Windows ping tests between pods and node
129+
for _, node := range nodes.Items {
130+
t.Log("Windows ping tests (1)")
131+
nodeIP := ""
132+
for _, address := range node.Status.Addresses {
133+
if address.Type == "InternalIP" {
134+
nodeIP = address.Address
135+
// Multiple addresses exist, break once Internal IP found.
136+
// Cannot call directly
137+
break
138+
}
139+
}
140+
141+
err := datapath.WindowsPodToNode(ctx, clientset, node.Name, nodeIP, *podNamespace, podLabelSelector, restConfig)
142+
require.NoError(t, err, "Windows pod to node, ping test failed with: %+v", err)
143+
t.Logf("Windows pod to node, passed for node: %s", node.Name)
144+
}
145+
})
146+
147+
t.Run("Windows ping tests pod -> pod", func(t *testing.T) {
148+
// Pod to pod same node
149+
for _, node := range nodes.Items {
150+
if node.Status.NodeInfo.OperatingSystem == string(apiv1.Windows) {
151+
t.Log("Windows ping tests (2) - Same Node")
152+
err := datapath.WindowsPodToPodPingTestSameNode(ctx, clientset, node.Name, *podNamespace, podLabelSelector, restConfig)
153+
require.NoError(t, err, "Windows pod to pod, same node, ping test failed with: %+v", err)
154+
t.Logf("Windows pod to windows pod, same node, passed for node: %s", node.ObjectMeta.Name)
155+
}
156+
}
157+
158+
// Pod to pod different node
159+
for i := 0; i < len(nodes.Items); i++ {
160+
t.Log("Windows ping tests (2) - Different Node")
161+
firstNode := nodes.Items[i%2].Name
162+
secondNode := nodes.Items[(i+1)%2].Name
163+
err = datapath.WindowsPodToPodPingTestDiffNode(ctx, clientset, firstNode, secondNode, *podNamespace, podLabelSelector, restConfig)
164+
require.NoError(t, err, "Windows pod to pod, different node, ping test failed with: %+v", err)
165+
t.Logf("Windows pod to windows pod, different node, passed for node: %s -> %s", firstNode, secondNode)
166+
167+
}
168+
})
169+
170+
t.Run("Windows url tests pod -> internet", func(t *testing.T) {
171+
// From windows pod, IWR a URL
172+
for _, node := range nodes.Items {
173+
if node.Status.NodeInfo.OperatingSystem == string(apiv1.Windows) {
174+
t.Log("Windows ping tests (3) - Pod to Internet tests")
175+
err := datapath.WindowsPodToInternet(ctx, clientset, node.Name, *podNamespace, podLabelSelector, restConfig)
176+
require.NoError(t, err, "Windows pod to internet test failed with: %+v", err)
177+
t.Logf("Windows pod to Internet url tests")
178+
}
179+
}
180+
})
181+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: windows-pod
5+
namespace: datapath-win
6+
spec:
7+
replicas: 4
8+
selector:
9+
matchLabels:
10+
app: datapod
11+
template:
12+
metadata:
13+
labels:
14+
app: datapod
15+
spec:
16+
containers:
17+
- name: windows-container
18+
image: mcr.microsoft.com/dotnet/framework/samples:aspnetapp
19+
command: ["powershell"]
20+
args: ["sleep", "5000"]
21+
nodeSelector:
22+
kubernetes.io/os: windows
+178
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
package datapath
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
8+
"github.com/Azure/azure-container-networking/test/internal/k8sutils"
9+
"github.com/pkg/errors"
10+
"github.com/sirupsen/logrus"
11+
apiv1 "k8s.io/api/core/v1"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/client-go/kubernetes"
14+
restclient "k8s.io/client-go/rest"
15+
)
16+
17+
func podTest(ctx context.Context, clientset *kubernetes.Clientset, srcPod *apiv1.Pod, cmd []string, rc *restclient.Config, passFunc func(string) error) error {
18+
logrus.Infof("podTest() - %v %v", srcPod.Name, cmd)
19+
output, err := k8sutils.ExecCmdOnPod(ctx, clientset, srcPod.Namespace, srcPod.Name, cmd, rc)
20+
if err != nil {
21+
return errors.Wrapf(err, "failed to execute command on pod: %v", srcPod.Name)
22+
}
23+
return passFunc(string(output))
24+
}
25+
26+
func WindowsPodToPodPingTestSameNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName, podNamespace, labelSelector string, rc *restclient.Config) error {
27+
logrus.Infof("Get Pods for Node: %s", nodeName)
28+
pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName)
29+
if err != nil {
30+
logrus.Error(err)
31+
return errors.Wrap(err, "k8s api call")
32+
}
33+
if len(pods.Items) <= 1 {
34+
return errors.New("Less than 2 pods on node")
35+
}
36+
37+
// Get first pod on this node
38+
firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{})
39+
if err != nil {
40+
return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err))
41+
}
42+
logrus.Infof("First pod: %v %v", firstPod.Name, firstPod.Status.PodIP)
43+
44+
// Get the second pod on this node
45+
secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[1].Name, metav1.GetOptions{})
46+
if err != nil {
47+
return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err))
48+
}
49+
logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP)
50+
51+
// Ping the second pod from the first pod
52+
return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows)
53+
}
54+
55+
func WindowsPodToPodPingTestDiffNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName1, nodeName2, podNamespace, labelSelector string, rc *restclient.Config) error {
56+
logrus.Infof("Get Pods for Node 1: %s", nodeName1)
57+
// Node 1
58+
pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName1)
59+
if err != nil {
60+
logrus.Error(err)
61+
return errors.Wrap(err, "k8s api call")
62+
}
63+
firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{})
64+
if err != nil {
65+
return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err))
66+
}
67+
logrus.Infof("First pod: %v %v", firstPod.Name, firstPod.Status.PodIP)
68+
69+
logrus.Infof("Get Pods for Node 2: %s", nodeName2)
70+
// Node 2
71+
pods, err = k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName2)
72+
if err != nil {
73+
logrus.Error(err)
74+
return errors.Wrap(err, "k8s api call")
75+
}
76+
secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{})
77+
if err != nil {
78+
return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err))
79+
}
80+
logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP)
81+
82+
// Ping the second pod from the first pod located on different nodes
83+
return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows)
84+
}
85+
86+
func WindowsPodToNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName, nodeIP, podNamespace, labelSelector string, rc *restclient.Config) error {
87+
logrus.Infof("Get Pods by Node: %s %s", nodeName, nodeIP)
88+
pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName)
89+
if err != nil {
90+
logrus.Error(err)
91+
return errors.Wrap(err, "k8s api call")
92+
}
93+
if len(pods.Items) <= 1 {
94+
return errors.New("Less than 2 pods on node")
95+
}
96+
// Get first pod on this node
97+
firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{})
98+
if err != nil {
99+
return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err))
100+
}
101+
logrus.Infof("First pod: %v", firstPod.Name)
102+
103+
// Get the second pod on this node
104+
secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[1].Name, metav1.GetOptions{})
105+
if err != nil {
106+
return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err))
107+
}
108+
logrus.Infof("Second pod: %v", secondPod.Name)
109+
110+
// Ping from pod to node
111+
resultOne := podTest(ctx, clientset, firstPod, []string{"ping", nodeIP}, rc, pingPassedWindows)
112+
resultTwo := podTest(ctx, clientset, secondPod, []string{"ping", nodeIP}, rc, pingPassedWindows)
113+
114+
if resultOne != nil {
115+
return resultOne
116+
}
117+
118+
if resultTwo != nil {
119+
return resultTwo
120+
}
121+
122+
return nil
123+
}
124+
125+
func WindowsPodToInternet(ctx context.Context, clientset *kubernetes.Clientset, nodeName, podNamespace, labelSelector string, rc *restclient.Config) error {
126+
logrus.Infof("Get Pods by Node: %s", nodeName)
127+
pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName)
128+
if err != nil {
129+
logrus.Error(err)
130+
return errors.Wrap(err, "k8s api call")
131+
}
132+
if len(pods.Items) <= 1 {
133+
return errors.New("Less than 2 pods on node")
134+
}
135+
136+
// Get first pod on this node
137+
firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{})
138+
if err != nil {
139+
return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err))
140+
}
141+
logrus.Infof("First pod: %v", firstPod.Name)
142+
143+
// Get the second pod on this node
144+
secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[1].Name, metav1.GetOptions{})
145+
if err != nil {
146+
return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err))
147+
}
148+
logrus.Infof("Second pod: %v", secondPod.Name)
149+
150+
resultOne := podTest(ctx, clientset, firstPod, []string{"powershell", "Invoke-WebRequest", "www.bing.com", "-UseBasicParsing"}, rc, webRequestPassedWindows)
151+
resultTwo := podTest(ctx, clientset, secondPod, []string{"powershell", "Invoke-WebRequest", "www.bing.com", "-UseBasicParsing"}, rc, webRequestPassedWindows)
152+
153+
if resultOne != nil {
154+
return resultOne
155+
}
156+
157+
if resultTwo != nil {
158+
return resultTwo
159+
}
160+
161+
return nil
162+
}
163+
164+
func webRequestPassedWindows(output string) error {
165+
const searchString = "200 OK"
166+
if strings.Contains(output, searchString) {
167+
return nil
168+
}
169+
return errors.Wrapf(errors.New("Output did not contain \"200 OK\""), "output was: %s", output)
170+
}
171+
172+
func pingPassedWindows(output string) error {
173+
const searchString = "0% loss"
174+
if strings.Contains(output, searchString) {
175+
return nil
176+
}
177+
return errors.Wrapf(errors.New("Ping did not contain\"0% loss\""), "output was: %s", output)
178+
}

0 commit comments

Comments
 (0)