From 18b8b0fd84c1af4a4311e35f192305964bdecb42 Mon Sep 17 00:00:00 2001 From: Enrique Llorente Date: Mon, 2 Sep 2024 15:40:02 +0200 Subject: [PATCH] kubevirt, e2e: UDN Signed-off-by: Enrique Llorente --- test/e2e/kubevirt.go | 146 ++++++++++++++++++++++++++++++----- test/e2e/kubevirt/console.go | 10 ++- 2 files changed, 135 insertions(+), 21 deletions(-) diff --git a/test/e2e/kubevirt.go b/test/e2e/kubevirt.go index dd7a124017b..a4d1e41866b 100644 --- a/test/e2e/kubevirt.go +++ b/test/e2e/kubevirt.go @@ -14,6 +14,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "github.com/ovn-org/ovn-kubernetes/test/e2e/diagnostics" "github.com/ovn-org/ovn-kubernetes/test/e2e/kubevirt" @@ -31,6 +32,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/util/retry" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + testutils "k8s.io/kubernetes/test/utils" utilnet "k8s.io/utils/net" "k8s.io/utils/pointer" crclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -301,11 +303,11 @@ var _ = Describe("Kubevirt Virtual Machines", func() { return ips } - checkPodHasIPsAtNetwork = func(netName string, expectedNumberOfAddresses int) func(Gomega, *corev1.Pod) { + checkPodHasIPsAtNetwork = func(netName, ifaceName string, expectedNumberOfAddresses int) func(Gomega, *corev1.Pod) { return func(g Gomega, pod *corev1.Pod) { GinkgoHelper() netStatus, err := podNetworkStatus(pod, func(status nadapi.NetworkStatus) bool { - return status.Name == netName + return status.Name == netName && status.Interface == ifaceName }) g.Expect(err).ToNot(HaveOccurred()) g.Expect(netStatus).To(HaveLen(1)) @@ -313,16 +315,24 @@ var _ = Describe("Kubevirt Virtual Machines", func() { } } - httpServerTestPodsMultusNetworkIPs = func(netName string) map[string][]string { + checkPodRunningReady = func() func(Gomega, *corev1.Pod) { + return func(g Gomega, pod *corev1.Pod) { + GinkgoHelper() + ok, err := testutils.PodRunningReady(pod) + Expect(err).ToNot(HaveOccurred()) + Expect(ok).To(BeTrue()) + } + } + + httpServerTestPodsMultusNetworkIPs = func(nadName string) map[string][]string { GinkgoHelper() ips := map[string][]string{} for _, pod := range httpServerTestPods { - netStatus, err := podNetworkStatus(pod, func(status nadapi.NetworkStatus) bool { - return status.Name == netName - }) - Expect(err).NotTo(HaveOccurred()) - Expect(netStatus).To(HaveLen(1)) - ips[pod.Name] = append(ips[pod.Name], netStatus[0].IPs...) + ovnPodAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName) + Expect(err).ToNot(HaveOccurred()) + for _, ipnet := range ovnPodAnnotation.IPs { + ips[pod.Name] = append(ips[pod.Name], ipnet.IP.String()) + } } return ips } @@ -650,6 +660,19 @@ var _ = Describe("Kubevirt Virtual Machines", func() { return addresses } + virtualMachineAddressesFromGuest = func(vmi *kubevirtv1.VirtualMachineInstance) []string { + GinkgoHelper() + addresses := waitVirtualMachineAddresses(vmi) + ips := []string{} + for _, address := range addresses { + if net.ParseIP(address.Ip).IsLinkLocalUnicast() { + continue + } + ips = append(ips, address.Ip) + } + return ips + } + fcosVMI = func(idx int, labels map[string]string, annotations map[string]string, nodeSelector map[string]string, networkSource kubevirtv1.NetworkSource, butane string) (*kubevirtv1.VirtualMachineInstance, error) { workingDirectory, err := os.Getwd() if err != nil { @@ -1112,7 +1135,7 @@ passwd: }), ) }) - Context("with secondary network and persistent ips configured", func() { + Context("with user defined networks and persistent ips configured", func() { type testCommand struct { description string cmd func() @@ -1171,6 +1194,22 @@ passwd: }, } + virtualMachineWithUDN = resourceCommand{ + description: "VirtualMachine with interface binding for UDN", + cmd: func() string { + var err error + vm, err = fcosVM(1, nil /*labels*/, nil /*annotations*/, nil, /*nodeSelector*/ + kubevirtv1.NetworkSource{ + Pod: &kubevirtv1.PodNetwork{}, + }, butane) + Expect(err).ToNot(HaveOccurred()) + vm.Spec.Template.Spec.Domain.Devices.Interfaces[0].Bridge = nil + vm.Spec.Template.Spec.Domain.Devices.Interfaces[0].Binding = &kubevirtv1.PluginBinding{Name: "passt"} + createVirtualMachine(vm) + return vm.Name + }, + } + virtualMachineInstance = resourceCommand{ description: "VirtualMachineInstance", cmd: func() string { @@ -1185,6 +1224,22 @@ passwd: return vmi.Name }, } + + virtualMachineInstanceWithUDN = resourceCommand{ + description: "VirtualMachineInstance with interface binding for UDN", + cmd: func() string { + var err error + vmi, err = fcosVMI(1, nil /*labels*/, nil /*annotations*/, nil, /*nodeSelector*/ + kubevirtv1.NetworkSource{ + Pod: &kubevirtv1.PodNetwork{}, + }, butane) + Expect(err).ToNot(HaveOccurred()) + vmi.Spec.Domain.Devices.Interfaces[0].Bridge = nil + vmi.Spec.Domain.Devices.Interfaces[0].Binding = &kubevirtv1.PluginBinding{Name: "passt"} + createVirtualMachineInstance(vmi) + return vmi.Name + }, + } filterOutIPv6 = func(ips map[string][]string) map[string][]string { filteredOutIPs := map[string][]string{} for podName, podIPs := range ips { @@ -1206,6 +1261,7 @@ passwd: resource resourceCommand test testCommand topology string + role string } DescribeTable("should keep ip", func(td testData) { netConfig := newNetworkAttachmentConfig( @@ -1215,6 +1271,7 @@ passwd: topology: td.topology, cidr: strings.Join([]string{cidrIPv4, cidrIPv6}, ","), allowPersistentIPs: true, + role: td.role, }) if td.topology == "localnet" { @@ -1237,9 +1294,18 @@ passwd: Expect(err).ToNot(HaveOccurred()) selectedNodes = workerNodeList.Items networkName := fmt.Sprintf("%s/%s", nad.Namespace, nad.Name) - prepareHTTPServerPods(map[string]string{ - "k8s.v1.cni.cncf.io/networks": fmt.Sprintf(`[{"name": %q}]`, nad.Name), - }, checkPodHasIPsAtNetwork(networkName, 2 /*expectedNumberOfAddresses*/)) + httpServerPodsAnnotations := map[string]string{} + if td.role != "primary" { + httpServerPodsAnnotations["k8s.v1.cni.cncf.io/networks"] = fmt.Sprintf(`[{"name": %q}]`, nad.Name) + } + var httpServerPodCondition func(Gomega, *corev1.Pod) + if td.role != "primary" { + httpServerPodCondition = checkPodHasIPsAtNetwork(networkName, "net1", 2 /*expectedNumberOfAddresses*/) + } else { + httpServerPodCondition = checkPodRunningReady() + } + + prepareHTTPServerPods(httpServerPodsAnnotations, httpServerPodCondition) vmiName := td.resource.cmd() vmi = &kubevirtv1.VirtualMachineInstance{ @@ -1252,12 +1318,23 @@ passwd: Expect(crClient.Get(context.TODO(), crclient.ObjectKeyFromObject(vmi), vmi)).To(Succeed()) step := by(vmi.Name, "Login to virtual machine for the first time") - Expect(kubevirt.LoginToFedora(vmi, "core", "fedora")).To(Succeed(), step) - expectedAddreses = virtualMachineAddressesFromStatus(vmi, 2 /*two addresses, dual stack*/) + if td.role != "primary" { + Expect(kubevirt.LoginToFedora(vmi, "core", "fedora")).To(Succeed(), step) + } else { + Expect(kubevirt.LoginToFedoraWithHostname(vmi, "core", "fedora", "localhost")).To(Succeed(), step) + } + + step = by(vmi.Name, "Wait for addresses at the virtual machine") + if td.role != "primary" { + expectedAddreses = virtualMachineAddressesFromStatus(vmi, 2 /*two addresses, dual stack*/) + } else { + expectedAddreses = virtualMachineAddressesFromGuest(vmi) + } step = by(vmi.Name, fmt.Sprintf("Check east/west traffic before %s %s", td.resource.description, td.test.description)) testPodsIPs := httpServerTestPodsMultusNetworkIPs(networkName) + //TODO: We do support it with primary, since passt do support it // kubevirt secondary IPAM do not support IPv6, so guest is not // going to have an ipv6 address at the interface testPodsIPs = filterOutIPv6(testPodsIPs) @@ -1268,15 +1345,30 @@ passwd: td.test.cmd() step = by(vm.Name, fmt.Sprintf("Login to virtual machine after %s %s", td.resource.description, td.test.description)) - Expect(kubevirt.LoginToFedora(vmi, "core", "fedora")).To(Succeed(), step) - obtainedAddresses := virtualMachineAddressesFromStatus(vmi, 2 /*two addresses, dual stack*/) + if td.role != "primary" { + Expect(kubevirt.LoginToFedora(vmi, "core", "fedora")).To(Succeed(), step) + } else { + Expect(kubevirt.LoginToFedoraWithHostname(vmi, "core", "fedora", "localhost")).To(Succeed(), step) + } + var obtainedAddresses []string + + if td.role != "primary" { + obtainedAddresses = virtualMachineAddressesFromStatus(vmi, 2 /*two addresses, dual stack*/) + } else { + obtainedAddresses = virtualMachineAddressesFromGuest(vmi) + } + Expect(obtainedAddresses).To(Equal(expectedAddreses)) step = by(vmi.Name, fmt.Sprintf("Check east/west traffic after %s %s", td.resource.description, td.test.description)) checkEastWestTraffic(vmi, testPodsIPs, step) }, func(td testData) string { - return fmt.Sprintf("after %s of %s with %s", td.test.description, td.resource.description, td.topology) + role := "secondary" + if td.role != "" { + role = td.role + } + return fmt.Sprintf("after %s of %s with %s/%s", td.test.description, td.resource.description, role, td.topology) }, Entry(nil, testData{ resource: virtualMachine, @@ -1288,6 +1380,12 @@ passwd: test: restart, topology: "layer2", }), + Entry(nil, testData{ + resource: virtualMachineWithUDN, + test: restart, + topology: "layer2", + role: "primary", + }), Entry(nil, testData{ resource: virtualMachine, test: liveMigrate, @@ -1298,6 +1396,12 @@ passwd: test: liveMigrate, topology: "layer2", }), + Entry(nil, testData{ + resource: virtualMachineWithUDN, + test: liveMigrate, + topology: "layer2", + role: "primary", + }), Entry(nil, testData{ resource: virtualMachineInstance, test: liveMigrate, @@ -1308,6 +1412,12 @@ passwd: test: liveMigrate, topology: "layer2", }), + Entry(nil, testData{ + resource: virtualMachineInstanceWithUDN, + test: liveMigrate, + topology: "layer2", + role: "primary", + }), ) }) }) diff --git a/test/e2e/kubevirt/console.go b/test/e2e/kubevirt/console.go index a8ee71c4421..822bd041620 100644 --- a/test/e2e/kubevirt/console.go +++ b/test/e2e/kubevirt/console.go @@ -182,8 +182,12 @@ func expectBatchWithValidatedSend(expecter expect.Expecter, batch []expect.Batch return res, err } -// LoginToFedora performs a console login to a Fedora base VM func LoginToFedora(vmi *kubevirtv1.VirtualMachineInstance, user, password string) error { + return LoginToFedoraWithHostname(vmi, user, password, vmi.Name) +} + +// LoginToFedora performs a console login to a Fedora base VM +func LoginToFedoraWithHostname(vmi *kubevirtv1.VirtualMachineInstance, user, password, hostname string) error { expecter, _, err := newExpecter(vmi, consoleConnectionTimeout, expect.Verbose(true), expect.VerboseWriter(GinkgoWriter)) if err != nil { return err @@ -197,7 +201,7 @@ func LoginToFedora(vmi *kubevirtv1.VirtualMachineInstance, user, password string // Do not login, if we already logged in loggedInPromptRegex := fmt.Sprintf( - `(\[%s@%s ~\]\$ |\[root@%s %s\]\# )`, user, vmi.Name, vmi.Name, user, + `(\[%s@%s ~\]\$ |\[root@%s %s\]\# )`, user, hostname, hostname, user, ) b := []expect.Batcher{ &expect.BSnd{S: "\n"}, @@ -215,7 +219,7 @@ func LoginToFedora(vmi *kubevirtv1.VirtualMachineInstance, user, password string &expect.Case{ // Using only "login: " would match things like "Last failed login: Tue Jun 9 22:25:30 UTC 2020 on ttyS0" // and in case the VM's did not get hostname form DHCP server try the default hostname - R: regexp.MustCompile(fmt.Sprintf(`%s login: `, vmi.Name)), + R: regexp.MustCompile(fmt.Sprintf(`%s login: `, hostname)), S: user + "\n", T: expect.Next(), Rt: 10,