diff --git a/tests/cnf/core/network/sriov/tests/qinq.go b/tests/cnf/core/network/sriov/tests/qinq.go index f6d5d547f..ea1e84907 100644 --- a/tests/cnf/core/network/sriov/tests/qinq.go +++ b/tests/cnf/core/network/sriov/tests/qinq.go @@ -50,6 +50,7 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn srIovNetworkDot1Q = "sriovnetwork-dot1q" srIovNetworkDPDKDot1AD = "sriovnetwork-dpdk-dot1ad" srIovNetworkDPDKDot1Q = "sriovnetwork-dpdk-dot1q" + srIovNetworkDPDKClient = "sriovnetwork-dpdk-client" srIovNetworkPromiscuous = "sriovnetwork-promiscuous" serverNameDPDKDot1ad = "server-dpdk-1ad" serverNameDPDKDot1q = "server-dpdk-1q" @@ -69,6 +70,7 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn intNet3 = "net3" intBond0 = "bond0.100" intelDeviceIDE810 = "1593" + intelDeviceIDE710 = "158b" testCmdNet2 = []string{"bash", "-c", "sleep 5; testcmd -interface net2 -protocol tcp " + "-port 4444 -listen"} testCmdNet2Net3 = []string{"bash", "-c", "sleep 5; testcmd -interface net2 -protocol tcp " + @@ -79,6 +81,8 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn tcpDumpReadFileCMD = []string{"bash", "-c", "tail -20 /tmp/tcpdump"} tcpDumpDot1ADOutput = "(ethertype 802\\.1Q-QinQ \\(0x88a8\\)).*?(ethertype 802\\.1Q, vlan 100)" tcpDumpDot1QOutput = "(ethertype 802\\.1Q \\(0x8100\\)).*?(ethertype 802\\.1Q, vlan 100)" + tcpDumpDot1QDPDKOutput = "(ethertype 802\\.1Q \\(0x8100\\)).*?(ethertype 802\\.1Q \\(0x8100\\), vlan 100)" + tcpDumpDot1ADDPDKOutput = "(ethertype 802\\.1Q-QinQ \\(0x88a8\\)).*?(ethertype 802\\.1Q \\(0x8100\\), vlan 100)" tcpDumpDot1ADCVLAN101Output = "(ethertype 802\\.1Q-QinQ \\(0x88a8\\)).*?(ethertype 802\\.1Q, vlan 101)" tcpDumpDot1QCVLAN101QOutput = "(ethertype 802\\.1Q \\(0x8100\\)).*?(ethertype 802\\.1Q, vlan 101)" workerNodeList = []*nodes.Builder{} @@ -138,8 +142,8 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn } By("Define and create sriovnetwork Polices") - defineCreateSriovNetPolices(srIovPolicyNetDevice, srIovPolicyVfioPci, srIovPolicyResNameNetDevice, - srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0], sriovDeviceID) + defineCreateSriovNetPolices(srIovPolicyVfioPci, srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0], + sriovDeviceID) By("Define and create sriovnetworks") defineAndCreateSriovNetworks(srIovNetworkPromiscuous, srIovNetworkDot1AD, srIovNetworkDot1Q, srIovPolicyResNameNetDevice) @@ -488,8 +492,8 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn 24) Expect(err).ToNot(HaveOccurred(), "Fail to deploy PerformanceProfile") - defineCreateSriovNetPolices(srIovPolicyNetDevice, srIovPolicyVfioPci, srIovPolicyResNameNetDevice, - srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0], sriovDeviceID) + defineCreateSriovNetPolices(srIovPolicyVfioPci, srIovPolicyResNameVfioPci, srIovInterfacesUnderTest[0], + sriovDeviceID) By("Setting selinux flag container_use_devices to 1 on all compute nodes") err = cluster.ExecCmd(APIClient, NetConfig.WorkerLabel, "setsebool container_use_devices 1") @@ -497,18 +501,11 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn By("Define and create sriov-network with 802.1ad S-VLAN") defineAndCreateSrIovNetworkWithQinQ(srIovNetworkDPDKDot1AD, srIovPolicyResNameVfioPci, dot1ad) + defineAndCreateSrIovNetworkClientDPDK(srIovNetworkDPDKClient, srIovPolicyResNameVfioPci) By("Define and create sriov-network with 802.1q S-VLAN") defineAndCreateSrIovNetworkWithQinQ(srIovNetworkDPDKDot1Q, srIovPolicyResNameVfioPci, dot1q) - By("Define and create sriov-network for the promiscuous client") - - _, err := sriov.NewNetworkBuilder(APIClient, - srIovNetworkPromiscuous, NetConfig.SriovOperatorNamespace, tsparams.TestNamespaceName, - srIovPolicyResNameNetDevice).WithTrustFlag(true).Create() - Expect(err).ToNot(HaveOccurred(), - fmt.Sprintf("Failed to create sriov network srIovNetworkPromiscuous %s", err)) - By("Define and create a network attachment definition for dpdk container") tapNad, err := define.TapNad(APIClient, nadCVLANDpdk, tsparams.TestNamespaceName, 0, 0, nil) Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Fail to define the Network-Attachment-Definition %s", @@ -524,12 +521,6 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn if sriovDeviceID != intelDeviceIDE810 { Skip(fmt.Sprintf("The NIC %s does not support 802.1AD", sriovDeviceID)) } - By("Define and create a container in promiscuous mode") - tcpDumpContainer := createPromiscuousClient(workerNodeList[0].Definition.Name, - tcpDumpNet1CMD) - - By("Enable VF promiscuous support on sriov interface under test") - setVFPromiscMode(workerNodeList[0].Definition.Name, srIovInterfacesUnderTest[0], sriovDeviceID, "on") runQinQDpdkTestCases( workerNodeList[0].Definition.Name, @@ -537,15 +528,17 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn clientNameDPDKDot1ad, srIovNetworkDPDKDot1AD, nadCVLANDpdk, - tcpDumpDot1ADOutput, - tcpDumpContainer) + tcpDumpDot1ADDPDKOutput) }) It("Verify network traffic over a 802.1q QinQ tunnel between two DPDK pods on the same PF", reportxml.ID("72638"), func() { - By("Define and create a container in promiscuous mode") - tcpDumpContainer := createPromiscuousClient(workerNodeList[0].Definition.Name, - tcpDumpNet1CMD) + testOutPutString := tcpDumpDot1QDPDKOutput + if sriovDeviceID == intelDeviceIDE710 { + vlan, err := strconv.Atoi(NetConfig.VLAN) + Expect(err).ToNot(HaveOccurred(), "Failed to convert VLAN value") + testOutPutString = fmt.Sprintf("(ethertype 802\\.1Q \\(0x8100\\)).*?(vlan %d)", vlan) + } runQinQDpdkTestCases( workerNodeList[0].Definition.Name, @@ -553,8 +546,7 @@ var _ = Describe("QinQ", Ordered, Label(tsparams.LabelQinQTestCases), ContinueOn clientNameDPDKDot1q, srIovNetworkDPDKDot1Q, nadCVLANDpdk, - tcpDumpDot1QOutput, - tcpDumpContainer) + testOutPutString) }) AfterAll(func() { By("Removing performanceProfile") @@ -712,6 +704,19 @@ func defineAndCreateSrIovNetworkWithQinQ(srIovNetwork, resName, vlanProtocol str "Fail to pull NetworkAttachmentDefinition") } +func defineAndCreateSrIovNetworkClientDPDK(srIovNetworkName, resName string) { + srIovNetworkObject, err := sriov.NewNetworkBuilder( + APIClient, srIovNetworkName, NetConfig.SriovOperatorNamespace, tsparams.TestNamespaceName, resName).Create() + Expect(err).ToNot(HaveOccurred(), "Failed to create dpdk sriov-network") + + Eventually(func() bool { + _, err := nad.Pull(APIClient, srIovNetworkObject.Object.Name, tsparams.TestNamespaceName) + + return err == nil + }, tsparams.WaitTimeout, tsparams.RetryInterval).Should(BeTrue(), + "Fail to pull NetworkAttachmentDefinition") +} + func createPromiscuousClient(nodeName string, tcpDumpCMD []string) *pod.Builder { sriovNetworkDefault := pod.StaticIPAnnotation("sriovnetwork-promiscuous", []string{"192.168.100.1/24"}) @@ -889,10 +894,11 @@ func defineTestServerPmdCmd(ethPeer, pciAddress string) []string { return []string{"/bin/bash", "-c", baseCmd} } -func defineTestClientPmdCmd(ethPeer, pciAddress string) []string { +func defineTestClientPmdCmd(pciAddress string) []string { baseCmd := fmt.Sprintf( - "timeout -s SIGKILL 20 dpdk-testpmd --vdev=virtio_user0,path=/dev/vhost-net,iface=net2 -a %s -- "+ - "--forward-mode rxonly --eth-peer=0,%s --stats-period 5", pciAddress, ethPeer) + "timeout -s SIGKILL 20 dpdk-testpmd "+ + "--vdev=virtio_user0,path=/dev/vhost-net,queues=2,queue_size=1024,iface=net2 -a %s "+ + "-- --stats-period 5", pciAddress) return []string{baseCmd} } @@ -906,7 +912,7 @@ func defineAndCreateServerDPDKPod( securityContext := corev1.SecurityContext{ RunAsUser: &rootUser, Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW"}, + Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW", "NET_ADMIN"}, }, } @@ -941,31 +947,32 @@ func defineAndCreateClientDPDKPod( podName, nodeName string, serverPodNetConfig []*multus.NetworkSelectionElement) *pod.Builder { - var rootUser int64 + var rootUser = int64(0) securityContext := corev1.SecurityContext{ RunAsUser: &rootUser, Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW"}, + Add: []corev1.Capability{"IPC_LOCK", "SYS_RESOURCE", "NET_RAW", "NET_ADMIN"}, }, } + testCommand := []string{"bash", "-c", "tcpdump -i net2 -e > /tmp/tcpdump"} dpdkContainerCfg, err := pod.NewContainerBuilder(podName, NetConfig.DpdkTestContainer, []string{"/bin/bash", "-c", "sleep INF"}).WithSecurityContext(&securityContext). WithResourceLimit("2Gi", "1Gi", 4). - WithResourceRequest("2Gi", "1Gi", 4).WithEnvVar("RUN_TYPE", "testcmd").GetContainerCfg() + WithResourceRequest("2Gi", "1Gi", 4).WithEnvVar("RUN_TYPE", "testcmd"). + GetContainerCfg() Expect(err).ToNot(HaveOccurred(), "Fail to define client dpdk container") dpdkPod, err := pod.NewBuilder(APIClient, podName, tsparams.TestNamespaceName, - NetConfig.DpdkTestContainer).WithSecondaryNetwork(serverPodNetConfig). - DefineOnNode(nodeName).RedefineDefaultContainer(*dpdkContainerCfg).WithHugePages(). + NetConfig.DpdkTestContainer).WithSecondaryNetwork(serverPodNetConfig).DefineOnNode(nodeName). + RedefineDefaultContainer(*dpdkContainerCfg).WithHugePages().RedefineDefaultCMD(testCommand). CreateAndWaitUntilRunning(4 * time.Minute) Expect(err).ToNot(HaveOccurred(), "Fail to create a dpdk client pod") return dpdkPod } -func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, nadCVLANDpdk, outPutSubString string, - tcpDumpPod *pod.Builder) { +func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, nadCVLANDpdk, outPutSubString string) { By("Define and create a 802.1AD dpdk server container") annotation := pod.StaticIPAnnotationWithMacAddress(sriovNetworkName, []string{}, tsparams.ServerMacAddress) @@ -977,7 +984,7 @@ func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, na var annotationDpdk []*multus.NetworkSelectionElement - sVlan := pod.StaticIPAnnotationWithMacAddress(sriovNetworkName, []string{}, tsparams.ClientMacAddress) + sVlan := pod.StaticIPAnnotationWithMacAddress("sriovnetwork-dpdk-client", []string{}, tsparams.ClientMacAddress) cVlan := pod.StaticAnnotation(nadCVLANDpdk) annotationDpdk = append(annotationDpdk, sVlan[0], cVlan) clientDpdk := defineAndCreateClientDPDKPod(clientName, nodeName, annotationDpdk) @@ -986,13 +993,13 @@ func runQinQDpdkTestCases(nodeName, serverName, clientName, sriovNetworkName, na By("Validate dpdk_testpmd traffic from the server to the client using CVLAN100.") - clientRxCmd := defineTestClientPmdCmd(tsparams.ServerMacAddress, - "${PCIDEVICE_OPENSHIFT_IO_SRIOVPOLICYVFIOPCI}") + clientRxCmd := defineTestClientPmdCmd("${PCIDEVICE_OPENSHIFT_IO_SRIOVPOLICYVFIOPCI}") + err := cmd.RxTrafficOnClientPod(clientDpdk, clientRxCmd[0]) Expect(err).ToNot(HaveOccurred(), "The Receive traffic test on the the client pod failed") - time.Sleep(30 * time.Minute) + By("Validate that the TCP traffic is double tagged") - readAndValidateTCPDump(tcpDumpPod, []string{"bash", "-c", "tail -20 /tmp/tcpdump"}, outPutSubString) + readAndValidateTCPDump(clientDpdk, []string{"bash", "-c", "tail -20 /tmp/tcpdump"}, outPutSubString) } // defineBondNAD returns network attachment definition for a Bond interface. @@ -1008,21 +1015,9 @@ func defineQinQBondNAD(nadname, mode string) *nad.Builder { return createdNad } -func defineCreateSriovNetPolices(netDeviceName, vfioPCIName, netDeviceResName, vfioPCIResName, sriovInterface, - sriovDeviceID string) { - By("Define and create sriov network policy using worker node label with netDevice type netdevice") - - _, err := sriov.NewPolicyBuilder( - APIClient, - netDeviceName, - NetConfig.SriovOperatorNamespace, - netDeviceResName, - 10, - []string{fmt.Sprintf("%s#0-9", sriovInterface)}, - NetConfig.WorkerLabelMap).Create() - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create sriovnetwork policy %s", - netDeviceName)) +func defineCreateSriovNetPolices(vfioPCIName, vfioPCIResName, sriovInterface, + sriovDeviceID string) { By("Define and create sriov network policy using worker node label with netDevice type vfio-pci") sriovPolicy := sriov.NewPolicyBuilder( @@ -1030,21 +1025,22 @@ func defineCreateSriovNetPolices(netDeviceName, vfioPCIName, netDeviceResName, v vfioPCIName, NetConfig.SriovOperatorNamespace, vfioPCIResName, - 16, - []string{fmt.Sprintf("%s#10-15", sriovInterface)}, - NetConfig.WorkerLabelMap).WithDevType("vfio-pci") + 6, + []string{fmt.Sprintf("%s#0-5", sriovInterface)}, + NetConfig.WorkerLabelMap).WithVhostNet(true) if sriovDeviceID == netparam.MlxDeviceID { - _, err = sriovPolicy.WithRDMA(true).Create() + _, err := sriovPolicy.WithRDMA(true).WithDevType("netdevice").Create() + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create Mellanox sriovnetwork policy %s", + vfioPCIName)) } else { - _, err = sriovPolicy.Create() + _, err := sriovPolicy.WithDevType("vfio-pci").Create() + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create Intel sriovnetwork policy %s", + vfioPCIName)) } - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to create sriovnetwork policy %s", - vfioPCIName)) - By("Waiting until cluster MCP and SR-IOV are stable") - err = netenv.WaitForSriovAndMCPStable( + err := netenv.WaitForSriovAndMCPStable( APIClient, tsparams.MCOWaitTimeout, time.Minute, NetConfig.CnfMcpLabel, NetConfig.SriovOperatorNamespace) Expect(err).ToNot(HaveOccurred(), "Failed cluster is not stable") } @@ -1061,11 +1057,9 @@ func defineAndCreateSriovNetworks(sriovNetworkPromiscName, sriovNetworkDot1ADNam By("Define and create sriov-network with 802.1ad S-VLAN") defineAndCreateSrIovNetworkWithQinQ(sriovNetworkDot1ADName, sriovResName, "802.1ad") - Expect(err).ToNot(HaveOccurred(), "Failed to create sriov network srIovNetworkDot1AD") By("Define and create sriov-network with 802.1q S-VLAN") defineAndCreateSrIovNetworkWithQinQ(sriovNetworkDot1QName, sriovResName, "802.1q") - Expect(err).ToNot(HaveOccurred(), "Failed to create sriov network srIovNetworkDot1Q") } func defineAndCreateNADs(nadCVLAN100, nadCVLAN101, nadMasterBond0, intNet1 string) {