Skip to content

Commit

Permalink
cnf network: fix-dpdk-test-case
Browse files Browse the repository at this point in the history
  • Loading branch information
Gregory Kopels committed Jul 7, 2024
1 parent d61419b commit 9c583e4
Showing 1 changed file with 60 additions and 66 deletions.
126 changes: 60 additions & 66 deletions tests/cnf/core/network/sriov/tests/qinq.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
srIovNetworkDot1Q = "sriovnetwork-dot1q"
srIovNetworkDPDKDot1AD = "sriovnetwork-dpdk-dot1ad"
srIovNetworkDPDKDot1Q = "sriovnetwork-dpdk-dot1q"
srIovNetworkDPDKClient = "sriovnetwork-dpdk-client"
srIovNetworkPromiscuous = "sriovnetwork-promiscuous"
serverNameDPDKDot1ad = "server-dpdk-1ad"
serverNameDPDKDot1q = "server-dpdk-1q"
Expand All @@ -69,6 +70,7 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
intNet3 = "net3"
intBond0 = "bond0.100"
intelDeviceIDE810 = "1593"
intelDeviceIDE710 = "158b"
testCmdNet2 = []string{"bash", "-c", "sleep 5; testcmd -interface net2 -protocol tcp " +
"-port 4444 -listen"}
testCmdNet2Net3 = []string{"bash", "-c", "sleep 5; testcmd -interface net2 -protocol tcp " +
Expand All @@ -79,6 +81,8 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
tcpDumpReadFileCMD = []string{"bash", "-c", "tail -20 /tmp/tcpdump"}
tcpDumpDot1ADOutput = "(ethertype 802\\.1Q-QinQ \\(0x88a8\\)).*?(ethertype 802\\.1Q, vlan 100)"
tcpDumpDot1QOutput = "(ethertype 802\\.1Q \\(0x8100\\)).*?(ethertype 802\\.1Q, vlan 100)"
tcpDumpDot1QDPDKOutput = "(ethertype 802\\.1Q \\(0x8100\\)).*?(ethertype 802\\.1Q \\(0x8100\\), vlan 100)"
tcpDumpDot1ADDPDKOutput = "(ethertype 802\\.1Q-QinQ \\(0x88a8\\)).*?(ethertype 802\\.1Q \\(0x8100\\), vlan 100)"
tcpDumpDot1ADCVLAN101Output = "(ethertype 802\\.1Q-QinQ \\(0x88a8\\)).*?(ethertype 802\\.1Q, vlan 101)"
tcpDumpDot1QCVLAN101QOutput = "(ethertype 802\\.1Q \\(0x8100\\)).*?(ethertype 802\\.1Q, vlan 101)"
workerNodeList = []*nodes.Builder{}
Expand Down Expand Up @@ -138,8 +142,8 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
}

By("Define and create sriovnetwork Polices")
defineCreateSriovNetPolices(srIovPolicyNetDevice, srIovPolicyVfioPci, srIovPolicyResNameNetDevice,
srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0], sriovDeviceID)
defineCreateSriovNetPolices(srIovPolicyVfioPci, srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0],
sriovDeviceID)
By("Define and create sriovnetworks")
defineAndCreateSriovNetworks(srIovNetworkPromiscuous, srIovNetworkDot1AD, srIovNetworkDot1Q,
srIovPolicyResNameNetDevice)
Expand Down Expand Up @@ -488,27 +492,20 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
24)
Expect(err).ToNot(HaveOccurred(), "Fail to deploy PerformanceProfile")

defineCreateSriovNetPolices(srIovPolicyNetDevice, srIovPolicyVfioPci, srIovPolicyResNameNetDevice,
srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0], sriovDeviceID)
defineCreateSriovNetPolices(srIovPolicyVfioPci, srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0],
sriovDeviceID)

By("Setting selinux flag container_use_devices to 1 on all compute nodes")
err = cluster.ExecCmd(APIClient, NetConfig.WorkerLabel, "setsebool container_use_devices 1")
Expect(err).ToNot(HaveOccurred(), "Fail to enable selinux flag")

By("Define and create sriov-network with 802.1ad S-VLAN")
defineAndCreateSrIovNetworkWithQinQ(srIovNetworkDPDKDot1AD, srIovPolicyResNameVfioPci, dot1ad)
defineAndCreateSrIovNetworkClientDPDK(srIovNetworkDPDKClient, srIovPolicyResNameVfioPci)

By("Define and create sriov-network with 802.1q S-VLAN")
defineAndCreateSrIovNetworkWithQinQ(srIovNetworkDPDKDot1Q, srIovPolicyResNameVfioPci, dot1q)

By("Define and create sriov-network for the promiscuous client")

_, err := sriov.NewNetworkBuilder(APIClient,
srIovNetworkPromiscuous, NetConfig.SriovOperatorNamespace, tsparams.TestNamespaceName,
srIovPolicyResNameNetDevice).WithTrustFlag(true).Create()
Expect(err).ToNot(HaveOccurred(),
fmt.Sprintf("Failed to create sriov network srIovNetworkPromiscuous %s", err))

By("Define and create a network attachment definition for dpdk container")
tapNad, err := define.TapNad(APIClient, nadCVLANDpdk, tsparams.TestNamespaceName, 0, 0, nil)
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Fail to define the Network-Attachment-Definition %s",
Expand All @@ -524,37 +521,32 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn
if sriovDeviceID != intelDeviceIDE810 {
Skip(fmt.Sprintf("The NIC %s does not support 802.1AD", sriovDeviceID))
}
By("Define and create a container in promiscuous mode")
tcpDumpContainer := createPromiscuousClient(workerNodeList[0].Definition.Name,
tcpDumpNet1CMD)

By("Enable VF promiscuous support on sriov interface under test")
setVFPromiscMode(workerNodeList[0].Definition.Name, srIovInterfacesUnderTest[0], sriovDeviceID, "on")

runQinQDpdkTestCases(
workerNodeList[0].Definition.Name,
serverNameDPDKDot1ad,
clientNameDPDKDot1ad,
srIovNetworkDPDKDot1AD,
nadCVLANDpdk,
tcpDumpDot1ADOutput,
tcpDumpContainer)
tcpDumpDot1ADDPDKOutput)
})

It("Verify network traffic over a 802.1q QinQ tunnel between two DPDK pods on the same PF",
reportxml.ID("72638"), func() {
By("Define and create a container in promiscuous mode")
tcpDumpContainer := createPromiscuousClient(workerNodeList[0].Definition.Name,
tcpDumpNet1CMD)
testOutPutString := tcpDumpDot1QDPDKOutput
if sriovDeviceID == intelDeviceIDE710 {
vlan, err := strconv.Atoi(NetConfig.VLAN)
Expect(err).ToNot(HaveOccurred(), "Failed to convert VLAN value")
testOutPutString = fmt.Sprintf("(ethertype 802\\.1Q \\(0x8100\\)).*?(vlan %d)", vlan)
}

runQinQDpdkTestCases(
workerNodeList[0].Definition.Name,
serverNameDPDKDot1q,
clientNameDPDKDot1q,
srIovNetworkDPDKDot1Q,
nadCVLANDpdk,
tcpDumpDot1QOutput,
tcpDumpContainer)
testOutPutString)
})
AfterAll(func() {
By("Removing performanceProfile")
Expand Down Expand Up @@ -712,6 +704,19 @@ func defineAndCreateSrIovNetworkWithQinQ(srIovNetwork, resName, vlanProtocol str
"Fail to pull NetworkAttachmentDefinition")
}

func defineAndCreateSrIovNetworkClientDPDK(srIovNetworkName, resName string) {
srIovNetworkObject, err := sriov.NewNetworkBuilder(
APIClient, srIovNetworkName, NetConfig.SriovOperatorNamespace, tsparams.TestNamespaceName, resName).Create()
Expect(err).ToNot(HaveOccurred(), "Failed to create dpdk sriov-network")

Eventually(func() bool {
_, err := nad.Pull(APIClient, srIovNetworkObject.Object.Name, tsparams.TestNamespaceName)

return err == nil
}, tsparams.WaitTimeout, tsparams.RetryInterval).Should(BeTrue(),
"Fail to pull NetworkAttachmentDefinition")
}

func createPromiscuousClient(nodeName string, tcpDumpCMD []string) *pod.Builder {
sriovNetworkDefault := pod.StaticIPAnnotation("sriovnetwork-promiscuous", []string{"192.168.100.1/24"})

Expand Down Expand Up @@ -889,10 +894,11 @@ func defineTestServerPmdCmd(ethPeer, pciAddress string) []string {
return []string{"/bin/bash", "-c", baseCmd}
}

func defineTestClientPmdCmd(ethPeer, pciAddress string) []string {
func defineTestClientPmdCmd(pciAddress string) []string {
baseCmd := fmt.Sprintf(
"timeout -s SIGKILL 20 dpdk-testpmd --vdev=virtio_user0,path=/dev/vhost-net,iface=net2 -a %s -- "+
"--forward-mode rxonly --eth-peer=0,%s --stats-period 5", pciAddress, ethPeer)
"timeout -s SIGKILL 20 dpdk-testpmd "+
"--vdev=virtio_user0,path=/dev/vhost-net,queues=2,queue_size=1024,iface=net2 -a %s "+
"-- --stats-period 5", pciAddress)

return []string{baseCmd}
}
Expand All @@ -906,7 +912,7 @@ func defineAndCreateServerDPDKPod(
securityContext := corev1.SecurityContext{
RunAsUser: &rootUser,
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW"},
Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW", "NET_ADMIN"},
},
}

Expand Down Expand Up @@ -941,31 +947,32 @@ func defineAndCreateClientDPDKPod(
podName,
nodeName string,
serverPodNetConfig []*multus.NetworkSelectionElement) *pod.Builder {
var rootUser int64
var rootUser = int64(0)
securityContext := corev1.SecurityContext{
RunAsUser: &rootUser,
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW"},
Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW", "NET_ADMIN"},
},
}
testCommand := []string{"bash", "-c", "tcpdump -i net2 -e > /tmp/tcpdump"}

dpdkContainerCfg, err := pod.NewContainerBuilder(podName, NetConfig.DpdkTestContainer,
[]string{"/bin/bash", "-c", "sleep INF"}).WithSecurityContext(&securityContext).
WithResourceLimit("2Gi", "1Gi", 4).
WithResourceRequest("2Gi", "1Gi", 4).WithEnvVar("RUN_TYPE", "testcmd").GetContainerCfg()
WithResourceRequest("2Gi", "1Gi", 4).WithEnvVar("RUN_TYPE", "testcmd").
GetContainerCfg()
Expect(err).ToNot(HaveOccurred(), "Fail to define client dpdk container")

dpdkPod, err := pod.NewBuilder(APIClient, podName, tsparams.TestNamespaceName,
NetConfig.DpdkTestContainer).WithSecondaryNetwork(serverPodNetConfig).
DefineOnNode(nodeName).RedefineDefaultContainer(*dpdkContainerCfg).WithHugePages().
NetConfig.DpdkTestContainer).WithSecondaryNetwork(serverPodNetConfig).DefineOnNode(nodeName).
RedefineDefaultContainer(*dpdkContainerCfg).WithHugePages().RedefineDefaultCMD(testCommand).
CreateAndWaitUntilRunning(4 * time.Minute)
Expect(err).ToNot(HaveOccurred(), "Fail to create a dpdk client pod")

return dpdkPod
}

func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, nadCVLANDpdk, outPutSubString string,
tcpDumpPod *pod.Builder) {
func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, nadCVLANDpdk, outPutSubString string) {
By("Define and create a 802.1AD dpdk server container")

annotation := pod.StaticIPAnnotationWithMacAddress(sriovNetworkName, []string{}, tsparams.ServerMacAddress)
Expand All @@ -977,7 +984,7 @@ func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, na

var annotationDpdk []*multus.NetworkSelectionElement

sVlan := pod.StaticIPAnnotationWithMacAddress(sriovNetworkName, []string{}, tsparams.ClientMacAddress)
sVlan := pod.StaticIPAnnotationWithMacAddress("sriovnetwork-dpdk-client", []string{}, tsparams.ClientMacAddress)
cVlan := pod.StaticAnnotation(nadCVLANDpdk)
annotationDpdk = append(annotationDpdk, sVlan[0], cVlan)
clientDpdk := defineAndCreateClientDPDKPod(clientName, nodeName, annotationDpdk)
Expand All @@ -986,13 +993,13 @@ func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, na

By("Validate dpdk_testpmd traffic from the server to the client using CVLAN100.")

clientRxCmd := defineTestClientPmdCmd(tsparams.ServerMacAddress,
"${PCIDEVICE_OPENSHIFT_IO_SRIOVPOLICYVFIOPCI}")
clientRxCmd := defineTestClientPmdCmd("${PCIDEVICE_OPENSHIFT_IO_SRIOVPOLICYVFIOPCI}")

err := cmd.RxTrafficOnClientPod(clientDpdk, clientRxCmd[0])
Expect(err).ToNot(HaveOccurred(), "The Receive traffic test on the the client pod failed")
time.Sleep(30 * time.Minute)

By("Validate that the TCP traffic is double tagged")
readAndValidateTCPDump(tcpDumpPod, []string{"bash", "-c", "tail -20 /tmp/tcpdump"}, outPutSubString)
readAndValidateTCPDump(clientDpdk, []string{"bash", "-c", "tail -20 /tmp/tcpdump"}, outPutSubString)
}

// defineBondNAD returns network attachment definition for a Bond interface.
Expand All @@ -1008,43 +1015,32 @@ func defineQinQBondNAD(nadname, mode string) *nad.Builder {
return createdNad
}

func defineCreateSriovNetPolices(netDeviceName, vfioPCIName, netDeviceResName, vfioPCIResName, sriovInterface,
sriovDeviceID string) {
By("Define and create sriov network policy using worker node label with netDevice type netdevice")

_, err := sriov.NewPolicyBuilder(
APIClient,
netDeviceName,
NetConfig.SriovOperatorNamespace,
netDeviceResName,
10,
[]string{fmt.Sprintf("%s#0-9", sriovInterface)},
NetConfig.WorkerLabelMap).Create()
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create sriovnetwork policy %s",
netDeviceName))
func defineCreateSriovNetPolices(vfioPCIName, vfioPCIResName, sriovInterface,

sriovDeviceID string) {
By("Define and create sriov network policy using worker node label with netDevice type vfio-pci")

sriovPolicy := sriov.NewPolicyBuilder(
APIClient,
vfioPCIName,
NetConfig.SriovOperatorNamespace,
vfioPCIResName,
16,
[]string{fmt.Sprintf("%s#10-15", sriovInterface)},
NetConfig.WorkerLabelMap).WithDevType("vfio-pci")
6,
[]string{fmt.Sprintf("%s#0-5", sriovInterface)},
NetConfig.WorkerLabelMap).WithVhostNet(true)
if sriovDeviceID == netparam.MlxDeviceID {
_, err = sriovPolicy.WithRDMA(true).Create()
_, err := sriovPolicy.WithRDMA(true).WithDevType("netdevice").Create()
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create Mellanox sriovnetwork policy %s",
vfioPCIName))
} else {
_, err = sriovPolicy.Create()
_, err := sriovPolicy.WithDevType("vfio-pci").Create()
Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create Intel sriovnetwork policy %s",
vfioPCIName))
}

Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create sriovnetwork policy %s",
vfioPCIName))

By("Waiting until cluster MCP and SR-IOV are stable")

err = netenv.WaitForSriovAndMCPStable(
err := netenv.WaitForSriovAndMCPStable(
APIClient, tsparams.MCOWaitTimeout, time.Minute, NetConfig.CnfMcpLabel, NetConfig.SriovOperatorNamespace)
Expect(err).ToNot(HaveOccurred(), "Failed cluster is not stable")
}
Expand All @@ -1061,11 +1057,9 @@ func defineAndCreateSriovNetworks(sriovNetworkPromiscName, sriovNetworkDot1ADNam

By("Define and create sriov-network with 802.1ad S-VLAN")
defineAndCreateSrIovNetworkWithQinQ(sriovNetworkDot1ADName, sriovResName, "802.1ad")
Expect(err).ToNot(HaveOccurred(), "Failed to create sriov network srIovNetworkDot1AD")

By("Define and create sriov-network with 802.1q S-VLAN")
defineAndCreateSrIovNetworkWithQinQ(sriovNetworkDot1QName, sriovResName, "802.1q")
Expect(err).ToNot(HaveOccurred(), "Failed to create sriov network srIovNetworkDot1Q")
}

func defineAndCreateNADs(nadCVLAN100, nadCVLAN101, nadMasterBond0, intNet1 string) {
Expand Down

0 comments on commit 9c583e4

Please sign in to comment.