Skip to content

Commit

Permalink
kubevirt, e2e: UDN
Browse files Browse the repository at this point in the history 
Signed-off-by: Enrique Llorente <[email protected]>
  • Loading branch information
@qinqon @maiqueb
qinqon authored and maiqueb committed Sep 6, 2024
1 parent 5741d7a commit 18b8b0f
Show file tree
Hide file tree
Showing 2 changed files with 135 additions and 21 deletions.
146 changes: 128 additions & 18 deletions test/e2e/kubevirt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
"github.com/ovn-org/ovn-kubernetes/test/e2e/diagnostics"
"github.com/ovn-org/ovn-kubernetes/test/e2e/kubevirt"

Expand All @@ -31,6 +32,7 @@ import (
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/retry"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
testutils "k8s.io/kubernetes/test/utils"
utilnet "k8s.io/utils/net"
"k8s.io/utils/pointer"
crclient "sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -301,28 +303,36 @@ var _ = Describe("Kubevirt Virtual Machines", func() {
return ips
}

checkPodHasIPsAtNetwork = func(netName string, expectedNumberOfAddresses int) func(Gomega, *corev1.Pod) {
checkPodHasIPsAtNetwork = func(netName, ifaceName string, expectedNumberOfAddresses int) func(Gomega, *corev1.Pod) {
return func(g Gomega, pod *corev1.Pod) {
GinkgoHelper()
netStatus, err := podNetworkStatus(pod, func(status nadapi.NetworkStatus) bool {
return status.Name == netName
return status.Name == netName && status.Interface == ifaceName
})
g.Expect(err).ToNot(HaveOccurred())
g.Expect(netStatus).To(HaveLen(1))
g.Expect(netStatus[0].IPs).To(HaveLen(expectedNumberOfAddresses))
}
}

httpServerTestPodsMultusNetworkIPs = func(netName string) map[string][]string {
checkPodRunningReady = func() func(Gomega, *corev1.Pod) {
return func(g Gomega, pod *corev1.Pod) {
GinkgoHelper()
ok, err := testutils.PodRunningReady(pod)
Expect(err).ToNot(HaveOccurred())
Expect(ok).To(BeTrue())
}
}

httpServerTestPodsMultusNetworkIPs = func(nadName string) map[string][]string {
GinkgoHelper()
ips := map[string][]string{}
for _, pod := range httpServerTestPods {
netStatus, err := podNetworkStatus(pod, func(status nadapi.NetworkStatus) bool {
return status.Name == netName
})
Expect(err).NotTo(HaveOccurred())
Expect(netStatus).To(HaveLen(1))
ips[pod.Name] = append(ips[pod.Name], netStatus[0].IPs...)
ovnPodAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, nadName)
Expect(err).ToNot(HaveOccurred())
for _, ipnet := range ovnPodAnnotation.IPs {
ips[pod.Name] = append(ips[pod.Name], ipnet.IP.String())
}
}
return ips
}
Expand Down Expand Up @@ -650,6 +660,19 @@ var _ = Describe("Kubevirt Virtual Machines", func() {
return addresses
}

virtualMachineAddressesFromGuest = func(vmi *kubevirtv1.VirtualMachineInstance) []string {
GinkgoHelper()
addresses := waitVirtualMachineAddresses(vmi)
ips := []string{}
for _, address := range addresses {
if net.ParseIP(address.Ip).IsLinkLocalUnicast() {
continue
}
ips = append(ips, address.Ip)
}
return ips
}

fcosVMI = func(idx int, labels map[string]string, annotations map[string]string, nodeSelector map[string]string, networkSource kubevirtv1.NetworkSource, butane string) (*kubevirtv1.VirtualMachineInstance, error) {
workingDirectory, err := os.Getwd()
if err != nil {
Expand Down Expand Up @@ -1112,7 +1135,7 @@ passwd:
}),
)
})
Context("with secondary network and persistent ips configured", func() {
Context("with user defined networks and persistent ips configured", func() {
type testCommand struct {
description string
cmd func()
Expand Down Expand Up @@ -1171,6 +1194,22 @@ passwd:
},
}

virtualMachineWithUDN = resourceCommand{
description: "VirtualMachine with interface binding for UDN",
cmd: func() string {
var err error
vm, err = fcosVM(1, nil /*labels*/, nil /*annotations*/, nil, /*nodeSelector*/
kubevirtv1.NetworkSource{
Pod: &kubevirtv1.PodNetwork{},
}, butane)
Expect(err).ToNot(HaveOccurred())
vm.Spec.Template.Spec.Domain.Devices.Interfaces[0].Bridge = nil
vm.Spec.Template.Spec.Domain.Devices.Interfaces[0].Binding = &kubevirtv1.PluginBinding{Name: "passt"}
createVirtualMachine(vm)
return vm.Name
},
}

virtualMachineInstance = resourceCommand{
description: "VirtualMachineInstance",
cmd: func() string {
Expand All @@ -1185,6 +1224,22 @@ passwd:
return vmi.Name
},
}

virtualMachineInstanceWithUDN = resourceCommand{
description: "VirtualMachineInstance with interface binding for UDN",
cmd: func() string {
var err error
vmi, err = fcosVMI(1, nil /*labels*/, nil /*annotations*/, nil, /*nodeSelector*/
kubevirtv1.NetworkSource{
Pod: &kubevirtv1.PodNetwork{},
}, butane)
Expect(err).ToNot(HaveOccurred())
vmi.Spec.Domain.Devices.Interfaces[0].Bridge = nil
vmi.Spec.Domain.Devices.Interfaces[0].Binding = &kubevirtv1.PluginBinding{Name: "passt"}
createVirtualMachineInstance(vmi)
return vmi.Name
},
}
filterOutIPv6 = func(ips map[string][]string) map[string][]string {
filteredOutIPs := map[string][]string{}
for podName, podIPs := range ips {
Expand All @@ -1206,6 +1261,7 @@ passwd:
resource resourceCommand
test testCommand
topology string
role string
}
DescribeTable("should keep ip", func(td testData) {
netConfig := newNetworkAttachmentConfig(
Expand All @@ -1215,6 +1271,7 @@ passwd:
topology: td.topology,
cidr: strings.Join([]string{cidrIPv4, cidrIPv6}, ","),
allowPersistentIPs: true,
role: td.role,
})

if td.topology == "localnet" {
Expand All @@ -1237,9 +1294,18 @@ passwd:
Expect(err).ToNot(HaveOccurred())
selectedNodes = workerNodeList.Items
networkName := fmt.Sprintf("%s/%s", nad.Namespace, nad.Name)
prepareHTTPServerPods(map[string]string{
"k8s.v1.cni.cncf.io/networks": fmt.Sprintf(`[{"name": %q}]`, nad.Name),
}, checkPodHasIPsAtNetwork(networkName, 2 /*expectedNumberOfAddresses*/))
httpServerPodsAnnotations := map[string]string{}
if td.role != "primary" {
httpServerPodsAnnotations["k8s.v1.cni.cncf.io/networks"] = fmt.Sprintf(`[{"name": %q}]`, nad.Name)
}
var httpServerPodCondition func(Gomega, *corev1.Pod)
if td.role != "primary" {
httpServerPodCondition = checkPodHasIPsAtNetwork(networkName, "net1", 2 /*expectedNumberOfAddresses*/)
} else {
httpServerPodCondition = checkPodRunningReady()
}

prepareHTTPServerPods(httpServerPodsAnnotations, httpServerPodCondition)

vmiName := td.resource.cmd()
vmi = &kubevirtv1.VirtualMachineInstance{
Expand All @@ -1252,12 +1318,23 @@ passwd:
Expect(crClient.Get(context.TODO(), crclient.ObjectKeyFromObject(vmi), vmi)).To(Succeed())

step := by(vmi.Name, "Login to virtual machine for the first time")
Expect(kubevirt.LoginToFedora(vmi, "core", "fedora")).To(Succeed(), step)
expectedAddreses = virtualMachineAddressesFromStatus(vmi, 2 /*two addresses, dual stack*/)
if td.role != "primary" {
Expect(kubevirt.LoginToFedora(vmi, "core", "fedora")).To(Succeed(), step)
} else {
Expect(kubevirt.LoginToFedoraWithHostname(vmi, "core", "fedora", "localhost")).To(Succeed(), step)
}

step = by(vmi.Name, "Wait for addresses at the virtual machine")
if td.role != "primary" {
expectedAddreses = virtualMachineAddressesFromStatus(vmi, 2 /*two addresses, dual stack*/)
} else {
expectedAddreses = virtualMachineAddressesFromGuest(vmi)
}

step = by(vmi.Name, fmt.Sprintf("Check east/west traffic before %s %s", td.resource.description, td.test.description))
testPodsIPs := httpServerTestPodsMultusNetworkIPs(networkName)

//TODO: We do support it with primary, since passt do support it
// kubevirt secondary IPAM do not support IPv6, so guest is not
// going to have an ipv6 address at the interface
testPodsIPs = filterOutIPv6(testPodsIPs)
Expand All @@ -1268,15 +1345,30 @@ passwd:
td.test.cmd()

step = by(vm.Name, fmt.Sprintf("Login to virtual machine after %s %s", td.resource.description, td.test.description))
Expect(kubevirt.LoginToFedora(vmi, "core", "fedora")).To(Succeed(), step)
obtainedAddresses := virtualMachineAddressesFromStatus(vmi, 2 /*two addresses, dual stack*/)
if td.role != "primary" {
Expect(kubevirt.LoginToFedora(vmi, "core", "fedora")).To(Succeed(), step)
} else {
Expect(kubevirt.LoginToFedoraWithHostname(vmi, "core", "fedora", "localhost")).To(Succeed(), step)
}
var obtainedAddresses []string

if td.role != "primary" {
obtainedAddresses = virtualMachineAddressesFromStatus(vmi, 2 /*two addresses, dual stack*/)
} else {
obtainedAddresses = virtualMachineAddressesFromGuest(vmi)
}

Expect(obtainedAddresses).To(Equal(expectedAddreses))

step = by(vmi.Name, fmt.Sprintf("Check east/west traffic after %s %s", td.resource.description, td.test.description))
checkEastWestTraffic(vmi, testPodsIPs, step)
},
func(td testData) string {
return fmt.Sprintf("after %s of %s with %s", td.test.description, td.resource.description, td.topology)
role := "secondary"
if td.role != "" {
role = td.role
}
return fmt.Sprintf("after %s of %s with %s/%s", td.test.description, td.resource.description, role, td.topology)
},
Entry(nil, testData{
resource: virtualMachine,
Expand All @@ -1288,6 +1380,12 @@ passwd:
test: restart,
topology: "layer2",
}),
Entry(nil, testData{
resource: virtualMachineWithUDN,
test: restart,
topology: "layer2",
role: "primary",
}),
Entry(nil, testData{
resource: virtualMachine,
test: liveMigrate,
Expand All @@ -1298,6 +1396,12 @@ passwd:
test: liveMigrate,
topology: "layer2",
}),
Entry(nil, testData{
resource: virtualMachineWithUDN,
test: liveMigrate,
topology: "layer2",
role: "primary",
}),
Entry(nil, testData{
resource: virtualMachineInstance,
test: liveMigrate,
Expand All @@ -1308,6 +1412,12 @@ passwd:
test: liveMigrate,
topology: "layer2",
}),
Entry(nil, testData{
resource: virtualMachineInstanceWithUDN,
test: liveMigrate,
topology: "layer2",
role: "primary",
}),
)
})
})
10 changes: 7 additions & 3 deletions test/e2e/kubevirt/console.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,12 @@ func expectBatchWithValidatedSend(expecter expect.Expecter, batch []expect.Batch
return res, err
}

// LoginToFedora performs a console login to a Fedora base VM
func LoginToFedora(vmi *kubevirtv1.VirtualMachineInstance, user, password string) error {
return LoginToFedoraWithHostname(vmi, user, password, vmi.Name)
}

// LoginToFedora performs a console login to a Fedora base VM
func LoginToFedoraWithHostname(vmi *kubevirtv1.VirtualMachineInstance, user, password, hostname string) error {
expecter, _, err := newExpecter(vmi, consoleConnectionTimeout, expect.Verbose(true), expect.VerboseWriter(GinkgoWriter))
if err != nil {
return err
Expand All @@ -197,7 +201,7 @@ func LoginToFedora(vmi *kubevirtv1.VirtualMachineInstance, user, password string

// Do not login, if we already logged in
loggedInPromptRegex := fmt.Sprintf(
`(\[%s@%s ~\]\$ |\[root@%s %s\]\# )`, user, vmi.Name, vmi.Name, user,
`(\[%s@%s ~\]\$ |\[root@%s %s\]\# )`, user, hostname, hostname, user,
)
b := []expect.Batcher{
&expect.BSnd{S: "\n"},
Expand All @@ -215,7 +219,7 @@ func LoginToFedora(vmi *kubevirtv1.VirtualMachineInstance, user, password string
&expect.Case{
// Using only "login: " would match things like "Last failed login: Tue Jun 9 22:25:30 UTC 2020 on ttyS0"
// and in case the VM's did not get hostname form DHCP server try the default hostname
R: regexp.MustCompile(fmt.Sprintf(`%s login: `, vmi.Name)),
R: regexp.MustCompile(fmt.Sprintf(`%s login: `, hostname)),
S: user + "\n",
T: expect.Next(),
Rt: 10,
Expand Down

0 comments on commit 18b8b0f

Please sign in to comment.