Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cnf network: fix-dpdk-test-case #76

Merged
merged 1 commit into from
Jul 8, 2024
Merged

cnf network: fix-dpdk-test-case #76

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 60 additions & 66 deletions tests/cnf/core/network/sriov/tests/qinq.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
srIovNetworkDot1Q = "sriovnetwork-dot1q"
srIovNetworkDPDKDot1AD = "sriovnetwork-dpdk-dot1ad"
srIovNetworkDPDKDot1Q = "sriovnetwork-dpdk-dot1q"
srIovNetworkDPDKClient = "sriovnetwork-dpdk-client"
srIovNetworkPromiscuous = "sriovnetwork-promiscuous"
serverNameDPDKDot1ad = "server-dpdk-1ad"
serverNameDPDKDot1q = "server-dpdk-1q"
Expand All @@ -69,6 +70,7 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
intNet3 = "net3"
intBond0 = "bond0.100"
intelDeviceIDE810 = "1593"
intelDeviceIDE710 = "158b"
testCmdNet2 = []string{"bash", "-c", "sleep 5; testcmd -interface net2 -protocol tcp " +
"-port 4444 -listen"}
testCmdNet2Net3 = []string{"bash", "-c", "sleep 5; testcmd -interface net2 -protocol tcp " +
Expand All @@ -79,6 +81,8 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
tcpDumpReadFileCMD = []string{"bash", "-c", "tail -20 /tmp/tcpdump"}
tcpDumpDot1ADOutput = "(ethertype 802\\.1Q-QinQ \\(0x88a8\\)).*?(ethertype 802\\.1Q, vlan 100)"
tcpDumpDot1QOutput = "(ethertype 802\\.1Q \\(0x8100\\)).*?(ethertype 802\\.1Q, vlan 100)"
tcpDumpDot1QDPDKOutput = "(ethertype 802\\.1Q \\(0x8100\\)).*?(ethertype 802\\.1Q \\(0x8100\\), vlan 100)"
tcpDumpDot1ADDPDKOutput = "(ethertype 802\\.1Q-QinQ \\(0x88a8\\)).*?(ethertype 802\\.1Q \\(0x8100\\), vlan 100)"
tcpDumpDot1ADCVLAN101Output = "(ethertype 802\\.1Q-QinQ \\(0x88a8\\)).*?(ethertype 802\\.1Q, vlan 101)"
tcpDumpDot1QCVLAN101QOutput = "(ethertype 802\\.1Q \\(0x8100\\)).*?(ethertype 802\\.1Q, vlan 101)"
workerNodeList = []*nodes.Builder{}
Expand Down Expand Up @@ -138,8 +142,8 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
}

By("Define and create sriovnetwork Polices")
defineCreateSriovNetPolices(srIovPolicyNetDevice, srIovPolicyVfioPci, srIovPolicyResNameNetDevice,
srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0], sriovDeviceID)
defineCreateSriovNetPolices(srIovPolicyVfioPci, srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0],
sriovDeviceID)
By("Define and create sriovnetworks")
defineAndCreateSriovNetworks(srIovNetworkPromiscuous, srIovNetworkDot1AD, srIovNetworkDot1Q,
srIovPolicyResNameNetDevice)
Expand Down Expand Up @@ -488,27 +492,20 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
24)
Expect(err).ToNot(HaveOccurred(), "Fail to deploy PerformanceProfile")

defineCreateSriovNetPolices(srIovPolicyNetDevice, srIovPolicyVfioPci, srIovPolicyResNameNetDevice,
srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0], sriovDeviceID)
defineCreateSriovNetPolices(srIovPolicyVfioPci, srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0],
sriovDeviceID)

By("Setting selinux flag container_use_devices to 1 on all compute nodes")
err = cluster.ExecCmd(APIClient, NetConfig.WorkerLabel, "setsebool container_use_devices 1")
Expect(err).ToNot(HaveOccurred(), "Fail to enable selinux flag")

By("Define and create sriov-network with 802.1ad S-VLAN")
defineAndCreateSrIovNetworkWithQinQ(srIovNetworkDPDKDot1AD, srIovPolicyResNameVfioPci, dot1ad)
defineAndCreateSrIovNetworkClientDPDK(srIovNetworkDPDKClient, srIovPolicyResNameVfioPci)

By("Define and create sriov-network with 802.1q S-VLAN")
defineAndCreateSrIovNetworkWithQinQ(srIovNetworkDPDKDot1Q, srIovPolicyResNameVfioPci, dot1q)

By("Define and create sriov-network for the promiscuous client")

_, err := sriov.NewNetworkBuilder(APIClient,
srIovNetworkPromiscuous, NetConfig.SriovOperatorNamespace, tsparams.TestNamespaceName,
srIovPolicyResNameNetDevice).WithTrustFlag(true).Create()
Expect(err).ToNot(HaveOccurred(),
fmt.Sprintf("Failed to create sriov network srIovNetworkPromiscuous %s", err))

By("Define and create a network attachment definition for dpdk container")
tapNad, err := define.TapNad(APIClient, nadCVLANDpdk, tsparams.TestNamespaceName, 0, 0, nil)
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Fail to define the Network-Attachment-Definition %s",
Expand All @@ -524,37 +521,32 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
if sriovDeviceID != intelDeviceIDE810 {
Skip(fmt.Sprintf("The NIC %s does not support 802.1AD", sriovDeviceID))
}
By("Define and create a container in promiscuous mode")
tcpDumpContainer := createPromiscuousClient(workerNodeList[0].Definition.Name,
tcpDumpNet1CMD)

By("Enable VF promiscuous support on sriov interface under test")
setVFPromiscMode(workerNodeList[0].Definition.Name, srIovInterfacesUnderTest[0], sriovDeviceID, "on")

runQinQDpdkTestCases(
workerNodeList[0].Definition.Name,
serverNameDPDKDot1ad,
clientNameDPDKDot1ad,
srIovNetworkDPDKDot1AD,
nadCVLANDpdk,
tcpDumpDot1ADOutput,
tcpDumpContainer)
tcpDumpDot1ADDPDKOutput)
})

It("Verify network traffic over a 802.1q QinQ tunnel between two DPDK pods on the same PF",
reportxml.ID("72638"), func() {
By("Define and create a container in promiscuous mode")
tcpDumpContainer := createPromiscuousClient(workerNodeList[0].Definition.Name,
tcpDumpNet1CMD)
testOutPutString := tcpDumpDot1QDPDKOutput
if sriovDeviceID == intelDeviceIDE710 {
vlan, err := strconv.Atoi(NetConfig.VLAN)
Expect(err).ToNot(HaveOccurred(), "Failed to convert VLAN value")
testOutPutString = fmt.Sprintf("(ethertype 802\\.1Q \\(0x8100\\)).*?(vlan %d)", vlan)
}

runQinQDpdkTestCases(
workerNodeList[0].Definition.Name,
serverNameDPDKDot1q,
clientNameDPDKDot1q,
srIovNetworkDPDKDot1Q,
nadCVLANDpdk,
tcpDumpDot1QOutput,
tcpDumpContainer)
testOutPutString)
})
AfterAll(func() {
By("Removing performanceProfile")
Expand Down Expand Up @@ -712,6 +704,19 @@ func defineAndCreateSrIovNetworkWithQinQ(srIovNetwork, resName, vlanProtocol str
"Fail to pull NetworkAttachmentDefinition")
}

func defineAndCreateSrIovNetworkClientDPDK(srIovNetworkName, resName string) {
srIovNetworkObject, err := sriov.NewNetworkBuilder(
APIClient, srIovNetworkName, NetConfig.SriovOperatorNamespace, tsparams.TestNamespaceName, resName).Create()
Expect(err).ToNot(HaveOccurred(), "Failed to create dpdk sriov-network")

Eventually(func() bool {
_, err := nad.Pull(APIClient, srIovNetworkObject.Object.Name, tsparams.TestNamespaceName)

return err == nil
}, tsparams.WaitTimeout, tsparams.RetryInterval).Should(BeTrue(),
"Fail to pull NetworkAttachmentDefinition")
}

func createPromiscuousClient(nodeName string, tcpDumpCMD []string) *pod.Builder {
sriovNetworkDefault := pod.StaticIPAnnotation("sriovnetwork-promiscuous", []string{"192.168.100.1/24"})

Expand Down Expand Up @@ -889,10 +894,11 @@ func defineTestServerPmdCmd(ethPeer, pciAddress string) []string {
return []string{"/bin/bash", "-c", baseCmd}
}

func defineTestClientPmdCmd(ethPeer, pciAddress string) []string {
func defineTestClientPmdCmd(pciAddress string) []string {
baseCmd := fmt.Sprintf(
"timeout -s SIGKILL 20 dpdk-testpmd --vdev=virtio_user0,path=/dev/vhost-net,iface=net2 -a %s -- "+
"--forward-mode rxonly --eth-peer=0,%s --stats-period 5", pciAddress, ethPeer)
"timeout -s SIGKILL 20 dpdk-testpmd "+
"--vdev=virtio_user0,path=/dev/vhost-net,queues=2,queue_size=1024,iface=net2 -a %s "+
"-- --stats-period 5", pciAddress)

return []string{baseCmd}
}
Expand All @@ -906,7 +912,7 @@ func defineAndCreateServerDPDKPod(
securityContext := corev1.SecurityContext{
RunAsUser: &rootUser,
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW"},
Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW", "NET_ADMIN"},
},
}

Expand Down Expand Up @@ -941,31 +947,32 @@ func defineAndCreateClientDPDKPod(
podName,
nodeName string,
serverPodNetConfig []*multus.NetworkSelectionElement) *pod.Builder {
var rootUser int64
var rootUser = int64(0)
securityContext := corev1.SecurityContext{
RunAsUser: &rootUser,
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW"},
Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW", "NET_ADMIN"},
},
}
testCommand := []string{"bash", "-c", "tcpdump -i net2 -e > /tmp/tcpdump"}

dpdkContainerCfg, err := pod.NewContainerBuilder(podName, NetConfig.DpdkTestContainer,
[]string{"/bin/bash", "-c", "sleep INF"}).WithSecurityContext(&securityContext).
WithResourceLimit("2Gi", "1Gi", 4).
WithResourceRequest("2Gi", "1Gi", 4).WithEnvVar("RUN_TYPE", "testcmd").GetContainerCfg()
WithResourceRequest("2Gi", "1Gi", 4).WithEnvVar("RUN_TYPE", "testcmd").
GetContainerCfg()
Expect(err).ToNot(HaveOccurred(), "Fail to define client dpdk container")

dpdkPod, err := pod.NewBuilder(APIClient, podName, tsparams.TestNamespaceName,
NetConfig.DpdkTestContainer).WithSecondaryNetwork(serverPodNetConfig).
DefineOnNode(nodeName).RedefineDefaultContainer(*dpdkContainerCfg).WithHugePages().
NetConfig.DpdkTestContainer).WithSecondaryNetwork(serverPodNetConfig).DefineOnNode(nodeName).
RedefineDefaultContainer(*dpdkContainerCfg).WithHugePages().RedefineDefaultCMD(testCommand).
CreateAndWaitUntilRunning(4 * time.Minute)
Expect(err).ToNot(HaveOccurred(), "Fail to create a dpdk client pod")

return dpdkPod
}

func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, nadCVLANDpdk, outPutSubString string,
tcpDumpPod *pod.Builder) {
func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, nadCVLANDpdk, outPutSubString string) {
By("Define and create a 802.1AD dpdk server container")

annotation := pod.StaticIPAnnotationWithMacAddress(sriovNetworkName, []string{}, tsparams.ServerMacAddress)
Expand All @@ -977,7 +984,7 @@ func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, na

var annotationDpdk []*multus.NetworkSelectionElement

sVlan := pod.StaticIPAnnotationWithMacAddress(sriovNetworkName, []string{}, tsparams.ClientMacAddress)
sVlan := pod.StaticIPAnnotationWithMacAddress("sriovnetwork-dpdk-client", []string{}, tsparams.ClientMacAddress)
cVlan := pod.StaticAnnotation(nadCVLANDpdk)
annotationDpdk = append(annotationDpdk, sVlan[0], cVlan)
clientDpdk := defineAndCreateClientDPDKPod(clientName, nodeName, annotationDpdk)
Expand All @@ -986,13 +993,13 @@ func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, na

By("Validate dpdk_testpmd traffic from the server to the client using CVLAN100.")

clientRxCmd := defineTestClientPmdCmd(tsparams.ServerMacAddress,
"${PCIDEVICE_OPENSHIFT_IO_SRIOVPOLICYVFIOPCI}")
clientRxCmd := defineTestClientPmdCmd("${PCIDEVICE_OPENSHIFT_IO_SRIOVPOLICYVFIOPCI}")

err := cmd.RxTrafficOnClientPod(clientDpdk, clientRxCmd[0])
Expect(err).ToNot(HaveOccurred(), "The Receive traffic test on the the client pod failed")
time.Sleep(30 * time.Minute)

By("Validate that the TCP traffic is double tagged")
readAndValidateTCPDump(tcpDumpPod, []string{"bash", "-c", "tail -20 /tmp/tcpdump"}, outPutSubString)
readAndValidateTCPDump(clientDpdk, []string{"bash", "-c", "tail -20 /tmp/tcpdump"}, outPutSubString)
}

// defineBondNAD returns network attachment definition for a Bond interface.
Expand All @@ -1008,43 +1015,32 @@ func defineQinQBondNAD(nadname, mode string) *nad.Builder {
return createdNad
}

func defineCreateSriovNetPolices(netDeviceName, vfioPCIName, netDeviceResName, vfioPCIResName, sriovInterface,
sriovDeviceID string) {
By("Define and create sriov network policy using worker node label with netDevice type netdevice")

_, err := sriov.NewPolicyBuilder(
APIClient,
netDeviceName,
NetConfig.SriovOperatorNamespace,
netDeviceResName,
10,
[]string{fmt.Sprintf("%s#0-9", sriovInterface)},
NetConfig.WorkerLabelMap).Create()
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create sriovnetwork policy %s",
netDeviceName))
func defineCreateSriovNetPolices(vfioPCIName, vfioPCIResName, sriovInterface,

sriovDeviceID string) {
By("Define and create sriov network policy using worker node label with netDevice type vfio-pci")

sriovPolicy := sriov.NewPolicyBuilder(
APIClient,
vfioPCIName,
NetConfig.SriovOperatorNamespace,
vfioPCIResName,
16,
[]string{fmt.Sprintf("%s#10-15", sriovInterface)},
NetConfig.WorkerLabelMap).WithDevType("vfio-pci")
6,
[]string{fmt.Sprintf("%s#0-5", sriovInterface)},
NetConfig.WorkerLabelMap).WithVhostNet(true)
if sriovDeviceID == netparam.MlxDeviceID {
_, err = sriovPolicy.WithRDMA(true).Create()
_, err := sriovPolicy.WithRDMA(true).WithDevType("netdevice").Create()
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create Mellanox sriovnetwork policy %s",
vfioPCIName))
} else {
_, err = sriovPolicy.Create()
_, err := sriovPolicy.WithDevType("vfio-pci").Create()
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create Intel sriovnetwork policy %s",
vfioPCIName))
}

Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create sriovnetwork policy %s",
vfioPCIName))

By("Waiting until cluster MCP and SR-IOV are stable")

err = netenv.WaitForSriovAndMCPStable(
err := netenv.WaitForSriovAndMCPStable(
APIClient, tsparams.MCOWaitTimeout, time.Minute, NetConfig.CnfMcpLabel, NetConfig.SriovOperatorNamespace)
Expect(err).ToNot(HaveOccurred(), "Failed cluster is not stable")
}
Expand All @@ -1061,11 +1057,9 @@ func defineAndCreateSriovNetworks(sriovNetworkPromiscName, sriovNetworkDot1ADNam

By("Define and create sriov-network with 802.1ad S-VLAN")
defineAndCreateSrIovNetworkWithQinQ(sriovNetworkDot1ADName, sriovResName, "802.1ad")
Expect(err).ToNot(HaveOccurred(), "Failed to create sriov network srIovNetworkDot1AD")

By("Define and create sriov-network with 802.1q S-VLAN")
defineAndCreateSrIovNetworkWithQinQ(sriovNetworkDot1QName, sriovResName, "802.1q")
Expect(err).ToNot(HaveOccurred(), "Failed to create sriov network srIovNetworkDot1Q")
}

func defineAndCreateNADs(nadCVLAN100, nadCVLAN101, nadMasterBond0, intNet1 string) {
Expand Down
Loading