kubevirt, dhcp: Add DHCP with ARP Proxy to LSP

The hypershift workers use DHCP for IP configuration, this change configure the ipv4/ipv6 DHCP options from the VM's LSP with the cidr from switch subnet, harcode arp proxy IP as default gw and the dns server from kubernetes or openshift service, it also configure the "arp_proxy" option at the LSP. Signed-off-by: Enrique Llorente <[email protected]>
ovn-org · Mar 9, 2023 · aee8984 · aee8984
1 parent bae4e55
commit aee8984
Show file tree

Hide file tree

Showing 9 changed files with 325 additions and 1 deletion.
diff --git a/go-controller/pkg/kubevirt/dhcp.go b/go-controller/pkg/kubevirt/dhcp.go
@@ -0,0 +1,79 @@
+package kubevirt
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	utilnet "k8s.io/utils/net"
+
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
+)
+
+const (
+	dhcpLeaseTime = 3500
+)
+
+func ComposeDHCPOptions(k8scli clientset.Interface, hostname string, cidrs []*net.IPNet) (*nbdb.DHCPOptions, *nbdb.DHCPOptions, error) {
+	if len(cidrs) == 0 {
+		return nil, nil, fmt.Errorf("missing cidrs to compose dchp options")
+	}
+	if hostname == "" {
+		return nil, nil, fmt.Errorf("missing hostname to compose dchp options")
+	}
+	dnsServer, err := k8scli.CoreV1().Services("kube-system").Get(context.Background(), "kube-dns", metav1.GetOptions{})
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			return nil, nil, err
+		}
+		dnsServer, err = k8scli.CoreV1().Services("openshift-dns").Get(context.Background(), "dns-default", metav1.GetOptions{})
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	dnsServerIPv4, dnsServerIPv6 := sortServiceClusterIPs(dnsServer)
+	var dhcpv4Options, dhcpv6Options *nbdb.DHCPOptions
+	for _, cidr := range cidrs {
+		if utilnet.IsIPv4CIDR(cidr) {
+			dhcpv4Options = composeDHCPOptions(cidr.String(), ARPProxyIPv4, dnsServerIPv4, hostname)
+		} else if utilnet.IsIPv6CIDR(cidr) {
+			dhcpv6Options = composeDHCPOptions(cidr.String(), ARPProxyIPv6, dnsServerIPv6, hostname)
+		}
+	}
+	return dhcpv4Options, dhcpv6Options, nil
+}
+
+func sortServiceClusterIPs(svc *corev1.Service) (string, string) {
+	clusterIPv4 := ""
+	clusterIPv6 := ""
+	for _, clusterIP := range svc.Spec.ClusterIPs {
+		if utilnet.IsIPv4String(clusterIP) {
+			clusterIPv4 = clusterIP
+		} else if utilnet.IsIPv6String(clusterIP) {
+			clusterIPv6 = clusterIP
+		}
+	}
+	return clusterIPv4, clusterIPv6
+}
+
+func composeDHCPOptions(cidr string, arpProxyIP, dnsServer, hostname string) *nbdb.DHCPOptions {
+	serverMAC := util.IPAddrToHWAddr(net.ParseIP(arpProxyIP)).String()
+	return &nbdb.DHCPOptions{
+		Cidr: cidr,
+		Options: map[string]string{
+			"lease_time": fmt.Sprintf("%d", dhcpLeaseTime),
+			"router":     arpProxyIP,
+			"dns_server": dnsServer,
+			"server_id":  arpProxyIP,
+			"server_mac": serverMAC,
+			"hostname":   fmt.Sprintf("%q", hostname),
+		},
+	}
+}
diff --git a/go-controller/pkg/kubevirt/dhcp_test.go b/go-controller/pkg/kubevirt/dhcp_test.go
@@ -0,0 +1,113 @@
+package kubevirt
+
+import (
+	"net"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/extensions/table"
+	. "github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
+)
+
+var _ = Describe("Kubevirt", func() {
+	type dhcpTest struct {
+		cidrs               []string
+		hostname            string
+		dns                 *corev1.Service
+		hasError            bool
+		expectedIPv4Options *nbdb.DHCPOptions
+		expectedIPv6Options *nbdb.DHCPOptions
+	}
+	var (
+		svc = func(namespace string, name string, clusterIPs []string) *corev1.Service {
+			return &corev1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "kube-system",
+					Name:      "kube-dns",
+				},
+				Spec: corev1.ServiceSpec{
+					ClusterIPs: clusterIPs,
+				},
+			}
+		}
+		parseCIDR = func(cidr string) *net.IPNet {
+			_, parsedCIDR, err := net.ParseCIDR(cidr)
+			Expect(err).ToNot(HaveOccurred())
+			return parsedCIDR
+		}
+	)
+	DescribeTable("composing dhcp options", func(t dhcpTest) {
+		svcs := []corev1.Service{}
+		if t.dns != nil {
+			svcs = append(svcs, *t.dns)
+		}
+		fakeClient := fake.NewSimpleClientset(&corev1.ServiceList{
+			Items: svcs,
+		})
+		cidrs := []*net.IPNet{}
+		for _, cidr := range t.cidrs {
+			cidrs = append(cidrs, parseCIDR(cidr))
+		}
+		ipv4Options, ipv6Options, err := ComposeDHCPOptions(fakeClient, t.hostname, cidrs)
+		if t.hasError {
+			Expect(err).To(HaveOccurred())
+		} else {
+			Expect(err).ToNot(HaveOccurred())
+		}
+		Expect(ipv4Options).To(Equal(t.expectedIPv4Options))
+		Expect(ipv6Options).To(Equal(t.expectedIPv6Options))
+	},
+		Entry("IPv4 Single stack and k8s dns", dhcpTest{
+			cidrs:               []string{"192.168.25.0/24"},
+			hostname:            "foo1",
+			dns:                 svc("kube-system", "core-dns", []string{"192.167.23.44"}),
+			expectedIPv4Options: composeDHCPOptions("192.168.25.0/24", ARPProxyIPv4, "192.167.23.44", "foo1"),
+		}),
+		Entry("IPv6 Single stack and k8s dns", dhcpTest{
+			cidrs:               []string{"2002:0:0:1234::/64"},
+			hostname:            "foo1",
+			dns:                 svc("kube-system", "core-dns", []string{"2001:1:2:3:4:5:6:7"}),
+			expectedIPv6Options: composeDHCPOptions("2002:0:0:1234::/64", ARPProxyIPv6, "2001:1:2:3:4:5:6:7", "foo1"),
+		}),
+		Entry("Dual stack and k8s dns", dhcpTest{
+			cidrs:               []string{"192.168.25.0/24", "2002:0:0:1234::/64"},
+			hostname:            "foo1",
+			dns:                 svc("kube-system", "core-dns", []string{"192.167.23.44", "2001:1:2:3:4:5:6:7"}),
+			expectedIPv4Options: composeDHCPOptions("192.168.25.0/24", ARPProxyIPv4, "192.167.23.44", "foo1"),
+			expectedIPv6Options: composeDHCPOptions("2002:0:0:1234::/64", ARPProxyIPv6, "2001:1:2:3:4:5:6:7", "foo1"),
+		}),
+		Entry("IPv4 Single stack and openshift dns", dhcpTest{
+			cidrs:               []string{"192.168.25.0/24"},
+			hostname:            "foo1",
+			dns:                 svc("openshift-dns", "dns-default", []string{"192.167.23.44"}),
+			expectedIPv4Options: composeDHCPOptions("192.168.25.0/24", ARPProxyIPv4, "192.167.23.44", "foo1"),
+		}),
+		Entry("IPv6 Single stack and openshift dns", dhcpTest{
+			cidrs:               []string{"2002:0:0:1234::/64"},
+			hostname:            "foo1",
+			dns:                 svc("openshift-dns", "dns-default", []string{"2001:1:2:3:4:5:6:7"}),
+			expectedIPv6Options: composeDHCPOptions("2002:0:0:1234::/64", ARPProxyIPv6, "2001:1:2:3:4:5:6:7", "foo1"),
+		}),
+		Entry("Dual stack and k8s openshift ", dhcpTest{
+			cidrs:               []string{"192.168.25.0/24", "2002:0:0:1234::/64"},
+			hostname:            "foo1",
+			dns:                 svc("openshift-dns", "dns-default", []string{"192.167.23.44", "2001:1:2:3:4:5:6:7"}),
+			expectedIPv4Options: composeDHCPOptions("192.168.25.0/24", ARPProxyIPv4, "192.167.23.44", "foo1"),
+			expectedIPv6Options: composeDHCPOptions("2002:0:0:1234::/64", ARPProxyIPv6, "2001:1:2:3:4:5:6:7", "foo1"),
+		}),
+		Entry("No cidr should fail", dhcpTest{hasError: true}),
+		Entry("No dns should fail", dhcpTest{cidrs: []string{"192.168.3.0/24"}, hasError: true}),
+		Entry("No hostname should fail", dhcpTest{
+			hostname: "",
+			cidrs:    []string{"192.168.25.0/24"},
+			dns:      svc("kube-system", "core-dns", []string{"192.167.23.44"}),
+			hasError: true,
+		}),
+	)
+
+})
diff --git a/go-controller/pkg/kubevirt/kubevirt_suite_test.go b/go-controller/pkg/kubevirt/kubevirt_suite_test.go
@@ -0,0 +1,13 @@
+package kubevirt
+
+import (
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+)
+
+func TestClusterNode(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "Kubevirt Suite")
+}
diff --git a/go-controller/pkg/kubevirt/pod.go b/go-controller/pkg/kubevirt/pod.go
@@ -53,3 +53,24 @@ func FindIPConfigByVMLabel(client clientset.Interface, pod *corev1.Pod) (IPConfi
 	}
 	return ipConfig, nil
 }
+
+// PodIsLiveMigrationLeftOver return true if the pod is a finished
+// virt-launcher from a migration
+func PodIsLiveMigrationLeftOver(client clientset.Interface, pod *corev1.Pod) (bool, error) {
+	vmPods, err := FindPodsByVMLabel(client, pod)
+	if err != nil {
+		return false, err
+	}
+
+	for _, vmPod := range vmPods {
+		if vmPod.CreationTimestamp.After(pod.CreationTimestamp.Time) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func PodMatchesExternalIDs(pod *corev1.Pod, externalIDs map[string]string) bool {
+	return len(externalIDs) > 1 && externalIDs["namespace"] == pod.Namespace && externalIDs[VMLabel] == pod.Labels[VMLabel]
+}
diff --git a/go-controller/pkg/kubevirt/switch.go b/go-controller/pkg/kubevirt/switch.go
@@ -0,0 +1,18 @@
+package kubevirt
+
+import (
+	"net"
+	"strings"
+
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
+)
+
+const (
+	ARPProxyIPv4 = "169.254.1.1"
+	ARPProxyIPv6 = "d7b:6b4d:7b25:d22f::1"
+)
+
+func ARPProxyLSPOption() string {
+	mac := util.IPAddrToHWAddr(net.ParseIP(ARPProxyIPv4)).String()
+	return strings.Join([]string{mac, ARPProxyIPv4, ARPProxyIPv6}, " ")
+}
diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go
@@ -14,6 +14,7 @@ import (
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kubevirt"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
@@ -332,7 +333,10 @@ func (bnc *BaseNetworkController) createNodeLogicalSwitch(nodeName string, hostS
 		Name:      types.SwitchToRouterPrefix + switchName,
 		Type:      "router",
 		Addresses: []string{"router"},
-		Options:   map[string]string{"router-port": types.RouterToSwitchPrefix + switchName},
+		Options: map[string]string{
+			"router-port": types.RouterToSwitchPrefix + switchName,
+			"arp_proxy":   kubevirt.ARPProxyLSPOption(),
+		},
 	}
 	sw := nbdb.LogicalSwitch{Name: switchName}
 	err = libovsdbops.CreateOrUpdateLogicalSwitchPortsOnSwitch(bnc.nbClient, &sw, &logicalSwitchPort)

diff --git a/go-controller/pkg/ovn/kubevirt.go b/go-controller/pkg/ovn/kubevirt.go
@@ -2,10 +2,15 @@ package ovn
 
 import (
 	"context"
+	"fmt"
+	"net"
 
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kubevirt"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
 	corev1 "k8s.io/api/core/v1"
+	kapi "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -37,3 +42,60 @@ func (bnc *BaseNetworkController) ensureIPConfigForVM(pod *corev1.Pod) error {
 	}
 	return nil
 }
+
+func (oc *DefaultNetworkController) addDHCPOptions(pod *corev1.Pod, lsp *nbdb.LogicalSwitchPort) error {
+	_, subnetSwitchName, err := oc.getSwitchNames(pod)
+	if err != nil {
+		return err
+	}
+	var switchSubnets []*net.IPNet
+	if switchSubnets = oc.lsManager.GetSwitchSubnets(subnetSwitchName); switchSubnets == nil {
+		return fmt.Errorf("cannot retrieve subnet for assigning gateway routes switch: %s", subnetSwitchName)
+	}
+	// Fake router to delegate on proxy arp mechanism
+	vmName := pod.Labels[kubevirt.VMLabel]
+	dhcpv4Options, dhcpv6Options, err := kubevirt.ComposeDHCPOptions(oc.client, vmName, switchSubnets)
+	if err != nil {
+		return fmt.Errorf("failed composing DHCP options: %v", err)
+	}
+	if dhcpv4Options != nil {
+		dhcpv4Options.ExternalIDs = map[string]string{
+			"namespace":      pod.Namespace,
+			kubevirt.VMLabel: vmName,
+		}
+	}
+	if dhcpv6Options != nil {
+		dhcpv6Options.ExternalIDs = map[string]string{
+			"namespace":      pod.Namespace,
+			kubevirt.VMLabel: vmName,
+		}
+	}
+	err = libovsdbops.CreateOrUpdateDhcpv4Options(oc.nbClient, lsp, dhcpv4Options, dhcpv6Options)
+	if err != nil {
+		return fmt.Errorf("failed adding ovn operations to add DHCP v4 options: %v", err)
+	}
+	return nil
+}
+
+func (oc *DefaultNetworkController) deleteDHCPOptions(pod *kapi.Pod) error {
+	predicate := func(item *nbdb.DHCPOptions) bool {
+		return kubevirt.PodMatchesExternalIDs(pod, item.ExternalIDs)
+	}
+	return libovsdbops.DeleteDHCPOptionsWithPredicate(oc.nbClient, predicate)
+}
+
+func (oc *DefaultNetworkController) kubevirtCleanUp(pod *corev1.Pod) error {
+	if kubevirt.AllowPodBridgeNetworkLiveMigration(pod.Annotations) {
+		isLiveMigrationLefover, err := kubevirt.PodIsLiveMigrationLeftOver(oc.client, pod)
+		if err != nil {
+			return err
+		}
+
+		if !isLiveMigrationLefover {
+			if err := oc.deleteDHCPOptions(pod); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go
@@ -19,6 +19,7 @@ import (
 	egressqosfake "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/egressqos/v1/apis/clientset/versioned/fake"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kubevirt"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
 	lsm "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager"
@@ -309,6 +310,7 @@ func addNodeLogicalFlows(testData []libovsdbtest.TestData, expectedOVNClusterRou
 		Type: "router",
 		Options: map[string]string{
 			"router-port": types.RouterToSwitchPrefix + node.Name,
+			"arp_proxy":   kubevirt.ARPProxyLSPOption(),
 		},
 		Addresses: []string{"router"},
 	})

diff --git a/go-controller/pkg/ovn/pods.go b/go-controller/pkg/ovn/pods.go
@@ -10,6 +10,7 @@ import (
 	"github.com/ovn-org/libovsdb/ovsdb"
 	hotypes "github.com/ovn-org/ovn-kubernetes/go-controller/hybrid-overlay/pkg/types"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/config"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kubevirt"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/libovsdbops"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/metrics"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
@@ -109,6 +110,10 @@ func (oc *DefaultNetworkController) deleteLogicalPort(pod *kapi.Pod, portInfo *l
 		return err
 	}
 
+	if err := oc.kubevirtCleanUp(pod); err != nil {
+		return err
+	}
+
 	// do not remove SNATs/GW routes/IPAM for an IP address unless we have validated no other pod is using it
 	if pInfo == nil {
 		return nil
@@ -256,6 +261,13 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *kapi.Pod) (err error) {
 			return err
 		}
 	}
+
+	if kubevirt.AllowPodBridgeNetworkLiveMigration(pod.Annotations) {
+		if err := oc.addDHCPOptions(pod, lsp); err != nil {
+			return err
+		}
+	}
+
 	//observe the pod creation latency metric for newly created pods only
 	if newlyCreatedPort {
 		metrics.RecordPodCreated(pod, oc.NetInfo)