diff --git a/contrib/kind.sh b/contrib/kind.sh index c59c81f3a79..42253803001 100755 --- a/contrib/kind.sh +++ b/contrib/kind.sh @@ -853,7 +853,8 @@ create_ovn_kube_manifests() { --multi-network-enable="${ENABLE_MULTI_NET}" \ --ovnkube-metrics-scale-enable="${OVN_METRICS_SCALE_ENABLE}" \ --compact-mode="${OVN_COMPACT_MODE}" \ - --enable-interconnect="${OVN_ENABLE_INTERCONNECT}" + --enable-interconnect="${OVN_ENABLE_INTERCONNECT}" \ + --enable-multi-external-gateway=true popd } diff --git a/dist/images/daemonset.sh b/dist/images/daemonset.sh index f83b73b5244..129474ad22b 100755 --- a/dist/images/daemonset.sh +++ b/dist/images/daemonset.sh @@ -300,6 +300,9 @@ while [ "$1" != "" ]; do --enable-interconnect) OVN_ENABLE_INTERCONNECT=$VALUE ;; + --enable-multi-external-gateway) + OVN_ENABLE_MULTI_EXTERNAL_GATEWAY=$VALUE + ;; *) echo "WARNING: unknown parameter \"$PARAM\"" exit 1 @@ -456,6 +459,8 @@ ovnkube_compact_mode_enable=${COMPACT_MODE:-"false"} echo "ovnkube_compact_mode_enable: ${ovnkube_compact_mode_enable}" ovn_enable_interconnect=${OVN_ENABLE_INTERCONNECT} echo "ovn_enable_interconnect: ${ovn_enable_interconnect}" +ovn_enable_multi_external_gateway=${OVN_ENABLE_MULTI_EXTERNAL_GATEWAY} +echo "ovn_enable_multi_external_gateway: ${ovn_enable_multi_external_gateway}" ovn_image=${ovnkube_image} \ ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \ @@ -498,6 +503,7 @@ ovn_image=${ovnkube_image} \ ovn_disable_ovn_iface_id_ver=${ovn_disable_ovn_iface_id_ver} \ ovnkube_node_mgmt_port_netdev=${ovnkube_node_mgmt_port_netdev} \ ovn_enable_interconnect=${ovn_enable_interconnect} \ + ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \ ovnkube_app_name=ovnkube-node \ j2 ../templates/ovnkube-node.yaml.j2 -o ${output_dir}/ovnkube-node.yaml @@ -572,6 +578,7 @@ ovn_image=${ovnkube_image} \ ovn_stateless_netpol_enable=${ovn_netpol_acl_enable} \ ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \ ovn_unprivileged_mode=${ovn_unprivileged_mode} \ + ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \ j2 ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml ovn_image=${ovnkube_image} \ @@ -603,6 +610,7 @@ ovn_image=${ovnkube_image} \ ovn_gateway_mode=${ovn_gateway_mode} \ ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \ ovn_enable_interconnect=${ovn_enable_interconnect} \ + ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \ j2 ../templates/ovnkube-control-plane.yaml.j2 -o ${output_dir}/ovnkube-control-plane.yaml ovn_image=${image} \ @@ -683,6 +691,7 @@ ovn_image=${ovnkube_image} \ ovn_empty_lb_events=${ovn_empty_lb_events} \ ovn_loglevel_nb=${ovn_loglevel_nb} ovn_loglevel_sb=${ovn_loglevel_sb} \ ovn_enable_interconnect=${ovn_enable_interconnect} \ + ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \ j2 ../templates/ovnkube-single-node-zone.yaml.j2 -o ${output_dir}/ovnkube-single-node-zone.yaml ovn_image=${ovnkube_image} \ @@ -734,6 +743,7 @@ ovn_image=${ovnkube_image} \ ovn_empty_lb_events=${ovn_empty_lb_events} \ ovn_loglevel_nb=${ovn_loglevel_nb} ovn_loglevel_sb=${ovn_loglevel_sb} \ ovn_enable_interconnect=${ovn_enable_interconnect} \ + ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \ j2 ../templates/ovnkube-zone-controller.yaml.j2 -o ${output_dir}/ovnkube-zone-controller.yaml ovn_image=${image} \ diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh index 17494f10941..3f20330d7d5 100755 --- a/dist/images/ovnkube.sh +++ b/dist/images/ovnkube.sh @@ -84,6 +84,7 @@ fi # OVN_ENCAP_IP - encap IP to be used for OVN traffic on the node. mandatory in case ovnkube-node-mode=="dpu" # OVN_HOST_NETWORK_NAMESPACE - namespace to classify host network traffic for applying network policies # OVN_DISABLE_FORWARDING - disable forwarding on OVNK controlled interfaces +# OVN_ENABLE_MULTI_EXTERNAL_GATEWAY - enable multi external gateway for ovn-kubernetes # The argument to the command is the operation to be performed # ovn-master ovn-controller ovn-node display display_env ovn_debug @@ -239,6 +240,8 @@ ovn_ipfix_cache_active_timeout=${OVN_IPFIX_CACHE_ACTIVE_TIMEOUT:-} \ ovn_stateless_netpol_enable=${OVN_STATELESS_NETPOL_ENABLE:-false} #OVN_ENABLE_INTERCONNECT - enable interconnect with multiple zones ovn_enable_interconnect=${OVN_ENABLE_INTERCONNECT:-false} +#OVN_ENABLE_MULTI_EXTERNAL_GATEWAY - enable multi external gateway +ovn_enable_multi_external_gateway=${OVN_ENABLE_MULTI_EXTERNAL_GATEWAY:-false} # OVNKUBE_NODE_MODE - is the mode which ovnkube node operates ovnkube_node_mode=${OVNKUBE_NODE_MODE:-"full"} @@ -1125,6 +1128,12 @@ ovn-master() { fi echo "ovn_stateless_netpol_enable_flag: ${ovn_stateless_netpol_enable_flag}" + ovnkube_enable_multi_external_gateway_flag= + if [[ ${ovn_enable_multi_external_gateway} == "true" ]]; then + ovnkube_enable_multi_external_gateway_flag="--enable-multi-external-gateway" + fi + echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}" + init_node_flags= if [[ ${ovnkube_compact_mode_enable} == "true" ]]; then init_node_flags="--init-node ${K8S_NODE} --nodeport" @@ -1165,6 +1174,7 @@ ovn-master() { ${ovnkube_metrics_scale_enable_flag} \ ${multi_network_enabled_flag} \ ${ovn_stateless_netpol_enable_flag} \ + ${ovnkube_enable_multi_external_gateway_flag} \ --metrics-bind-address ${ovnkube_master_metrics_bind_address} \ --host-network-namespace ${ovn_host_network_namespace} & @@ -1332,6 +1342,12 @@ ovnkube-controller() { fi echo "ovnkube_enable_interconnect_flag: ${ovnkube_enable_interconnect_flag}" + ovnkube_enable_multi_external_gateway_flag= + if [[ ${ovn_enable_multi_external_gateway} == "true" ]]; then + ovnkube_enable_multi_external_gateway_flag="--enable-multi-external-gateway" + fi + echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}" + echo "=============== ovnkube-controller ========== MASTER ONLY" /usr/bin/ovnkube \ --init-ovnkube-controller ${K8S_NODE} \ @@ -1361,6 +1377,7 @@ ovnkube-controller() { ${ovnkube_config_duration_enable_flag} \ ${multi_network_enabled_flag} \ ${ovnkube_enable_interconnect_flag} \ + ${ovnkube_enable_multi_external_gateway_flag} \ --zone ${ovn_zone} \ --metrics-bind-address ${ovnkube_master_metrics_bind_address} \ --host-network-namespace ${ovn_host_network_namespace} & @@ -1445,6 +1462,12 @@ ovn-cluster-manager() { fi echo "ovnkube_enable_interconnect_flag: ${ovnkube_enable_interconnect_flag}" + ovnkube_enable_multi_external_gateway_flag= + if [[ ${ovn_enable_multi_external_gateway} == "true" ]]; then + ovnkube_enable_multi_external_gateway_flag="--enable-multi-external-gateway" + fi + echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}" + echo "=============== ovn-cluster-manager ========== MASTER ONLY" /usr/bin/ovnkube \ --init-cluster-manager ${K8S_NODE} \ @@ -1465,6 +1488,7 @@ ovn-cluster-manager() { ${multi_network_enabled_flag} \ ${egressservice_enabled_flag} \ ${ovnkube_enable_interconnect_flag} \ + ${ovnkube_enable_multi_external_gateway_flag} \ --metrics-bind-address ${ovnkube_cluster_manager_metrics_bind_address} \ --host-network-namespace ${ovn_host_network_namespace} & @@ -1726,6 +1750,12 @@ ovn-node() { ovn_zone=$(get_node_zone) echo "ovnkube-node's configured zone is ${ovn_zone}" + ovnkube_enable_multi_external_gateway_flag= + if [[ ${ovn_enable_multi_external_gateway} == "true" ]]; then + ovnkube_enable_multi_external_gateway_flag="--enable-multi-external-gateway" + fi + echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}" + if [[ $ovn_nbdb != "local" ]]; then ovn_dbs="--nb-address=${ovn_nbdb}" fi @@ -1776,6 +1806,7 @@ ovn-node() { --metrics-bind-address ${ovnkube_node_metrics_bind_address} \ ${ovnkube_node_mode_flag} \ ${egress_interface} \ + ${ovnkube_enable_multi_external_gateway_flag} \ ${ovnkube_enable_interconnect_flag} \ --zone ${ovn_zone} \ --host-network-namespace ${ovn_host_network_namespace} \ diff --git a/dist/templates/ovnkube-control-plane.yaml.j2 b/dist/templates/ovnkube-control-plane.yaml.j2 index 74d664f8cea..eeac6c6d79a 100644 --- a/dist/templates/ovnkube-control-plane.yaml.j2 +++ b/dist/templates/ovnkube-control-plane.yaml.j2 @@ -160,6 +160,8 @@ spec: key: host_network_namespace - name: OVN_ENABLE_INTERCONNECT value: "{{ ovn_enable_interconnect }}" + - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY + value: "{{ ovn_enable_multi_external_gateway }}" # end of container volumes: diff --git a/dist/templates/ovnkube-master.yaml.j2 b/dist/templates/ovnkube-master.yaml.j2 index 01301b5cf5b..23652d7d044 100644 --- a/dist/templates/ovnkube-master.yaml.j2 +++ b/dist/templates/ovnkube-master.yaml.j2 @@ -279,6 +279,8 @@ spec: value: "{{ ovn_acl_logging_rate_limit }}" - name: OVN_STATELESS_NETPOL_ENABLE value: "{{ ovn_stateless_netpol_enable }}" + - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY + value: "{{ ovn_enable_multi_external_gateway }}" - name: OVN_HOST_NETWORK_NAMESPACE valueFrom: configMapKeyRef: diff --git a/dist/templates/ovnkube-node.yaml.j2 b/dist/templates/ovnkube-node.yaml.j2 index 09c01d5c24c..6b15a2fff4e 100644 --- a/dist/templates/ovnkube-node.yaml.j2 +++ b/dist/templates/ovnkube-node.yaml.j2 @@ -219,6 +219,8 @@ spec: value: "{{ ovn_multi_network_enable }}" - name: OVN_ENABLE_INTERCONNECT value: "{{ ovn_enable_interconnect }}" + - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY + value: "{{ ovn_enable_multi_external_gateway }}" {% endif -%} {% if ovnkube_app_name=="ovnkube-node-dpu-host" -%} - name: OVNKUBE_NODE_MODE diff --git a/dist/templates/ovnkube-single-node-zone.yaml.j2 b/dist/templates/ovnkube-single-node-zone.yaml.j2 index 74054c09603..0ac705a1c66 100644 --- a/dist/templates/ovnkube-single-node-zone.yaml.j2 +++ b/dist/templates/ovnkube-single-node-zone.yaml.j2 @@ -340,6 +340,8 @@ spec: value: "{{ ovn_acl_logging_rate_limit }}" - name: OVN_ENABLE_INTERCONNECT value: "{{ ovn_enable_interconnect }}" + - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY + value: "{{ ovn_enable_multi_external_gateway }}" - name: OVN_HOST_NETWORK_NAMESPACE valueFrom: configMapKeyRef: @@ -524,6 +526,8 @@ spec: value: "local" - name: OVN_ENABLE_INTERCONNECT value: "{{ ovn_enable_interconnect }}" + - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY + value: "{{ ovn_enable_multi_external_gateway }}" readinessProbe: exec: diff --git a/dist/templates/ovnkube-zone-controller.yaml.j2 b/dist/templates/ovnkube-zone-controller.yaml.j2 index c2a20825b4f..536904f91d9 100644 --- a/dist/templates/ovnkube-zone-controller.yaml.j2 +++ b/dist/templates/ovnkube-zone-controller.yaml.j2 @@ -356,6 +356,8 @@ spec: value: "{{ ovn_acl_logging_rate_limit }}" - name: OVN_ENABLE_INTERCONNECT value: "{{ ovn_enable_interconnect }}" + - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY + value: "{{ ovn_enable_multi_external_gateway }}" - name: OVN_HOST_NETWORK_NAMESPACE valueFrom: configMapKeyRef: diff --git a/go-controller/pkg/config/config.go b/go-controller/pkg/config/config.go index d570a3e51fa..e4b5730ed4c 100644 --- a/go-controller/pkg/config/config.go +++ b/go-controller/pkg/config/config.go @@ -359,6 +359,7 @@ type OVNKubernetesFeatureConfig struct { EnableMultiNetworkPolicy bool `gcfg:"enable-multi-networkpolicy"` EnableStatelessNetPol bool `gcfg:"enable-stateless-netpol"` EnableInterconnect bool `gcfg:"enable-interconnect"` + EnableMultiExternalGateway bool `gcfg:"enable-multi-external-gateway"` } // GatewayMode holds the node gateway mode @@ -971,6 +972,12 @@ var OVNK8sFeatureFlags = []cli.Flag{ Destination: &cliConfig.OVNKubernetesFeature.EnableEgressService, Value: OVNKubernetesFeature.EnableEgressService, }, + &cli.BoolFlag{ + Name: "enable-multi-external-gateway", + Usage: "Configure to use AdminPolicyBasedExternalRoute CRD feature with ovn-kubernetes.", + Destination: &cliConfig.OVNKubernetesFeature.EnableMultiExternalGateway, + Value: OVNKubernetesFeature.EnableMultiExternalGateway, + }, } // K8sFlags capture Kubernetes-related options diff --git a/go-controller/pkg/config/config_test.go b/go-controller/pkg/config/config_test.go index 1b635cfd23a..ecf904f3bf4 100644 --- a/go-controller/pkg/config/config_test.go +++ b/go-controller/pkg/config/config_test.go @@ -219,6 +219,7 @@ egressip-node-healthcheck-port=1234 enable-multi-network=false enable-multi-networkpolicy=false enable-interconnect=false +enable-multi-external-gateway=false ` var newData string @@ -317,6 +318,7 @@ var _ = Describe("Config Operations", func() { gomega.Expect(OVNKubernetesFeature.EnableMultiNetwork).To(gomega.BeFalse()) gomega.Expect(OVNKubernetesFeature.EnableMultiNetworkPolicy).To(gomega.BeFalse()) gomega.Expect(OVNKubernetesFeature.EnableInterconnect).To(gomega.BeFalse()) + gomega.Expect(OVNKubernetesFeature.EnableMultiExternalGateway).To(gomega.BeFalse()) for _, a := range []OvnAuthConfig{OvnNorth, OvnSouth} { gomega.Expect(a.Scheme).To(gomega.Equal(OvnDBSchemeUnix)) @@ -555,6 +557,7 @@ var _ = Describe("Config Operations", func() { "enable-multi-network=true", "enable-multi-networkpolicy=true", "enable-interconnect=true", + "enable-multi-external-gateway=true", "zone=foo", ) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -634,6 +637,7 @@ var _ = Describe("Config Operations", func() { gomega.Expect(OVNKubernetesFeature.EgressIPNodeHealthCheckPort).To(gomega.Equal(1234)) gomega.Expect(OVNKubernetesFeature.EnableMultiNetwork).To(gomega.BeTrue()) gomega.Expect(OVNKubernetesFeature.EnableInterconnect).To(gomega.BeTrue()) + gomega.Expect(OVNKubernetesFeature.EnableMultiExternalGateway).To(gomega.BeTrue()) gomega.Expect(HybridOverlay.ClusterSubnets).To(gomega.Equal([]CIDRNetworkEntry{ {ovntest.MustParseIPNet("11.132.0.0/14"), 23}, })) @@ -724,6 +728,7 @@ var _ = Describe("Config Operations", func() { gomega.Expect(OVNKubernetesFeature.EnableMultiNetwork).To(gomega.BeTrue()) gomega.Expect(OVNKubernetesFeature.EnableMultiNetworkPolicy).To(gomega.BeTrue()) gomega.Expect(OVNKubernetesFeature.EnableInterconnect).To(gomega.BeTrue()) + gomega.Expect(OVNKubernetesFeature.EnableMultiExternalGateway).To(gomega.BeTrue()) gomega.Expect(HybridOverlay.ClusterSubnets).To(gomega.Equal([]CIDRNetworkEntry{ {ovntest.MustParseIPNet("11.132.0.0/14"), 23}, })) @@ -786,6 +791,7 @@ var _ = Describe("Config Operations", func() { "-enable-multi-network=true", "-enable-multi-networkpolicy=true", "-enable-interconnect=true", + "-enable-multi-external-gateway=true", "-healthz-bind-address=0.0.0.0:4321", "-zone=bar", } diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index b17b4ebf8d4..91a9bfb9604 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -136,15 +136,6 @@ func NewDefaultNodeNetworkController(cnnci *CommonNodeNetworkControllerInfo) (*D } } - nc.apbExternalRouteNodeController, err = apbroute.NewExternalNodeController( - cnnci.apbExternalRouteClient, - nc.watchFactory.PodCoreInformer(), - nc.watchFactory.NamespaceInformer(), - stopChan) - if err != nil { - return nil, err - } - nc.initRetryFrameworkForNode() return nc, nil @@ -936,11 +927,22 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error { c.Run(1) }() } - nc.wg.Add(1) - go func() { - defer nc.wg.Done() - nc.apbExternalRouteNodeController.Run(1) - }() + if config.OVNKubernetesFeature.EnableMultiExternalGateway { + nc.apbExternalRouteNodeController, err = apbroute.NewExternalNodeController( + nc.apbExternalRouteClient, + nc.watchFactory.PodCoreInformer(), + nc.watchFactory.NamespaceInformer(), + nc.stopChan) + if err != nil { + return err + } + + nc.wg.Add(1) + go func() { + defer nc.wg.Done() + nc.apbExternalRouteNodeController.Run(1) + }() + } nc.wg.Add(1) go func() { diff --git a/go-controller/pkg/ovn/default_network_controller.go b/go-controller/pkg/ovn/default_network_controller.go index 10a30972af9..ecdbbd5143f 100644 --- a/go-controller/pkg/ovn/default_network_controller.go +++ b/go-controller/pkg/ovn/default_network_controller.go @@ -514,11 +514,13 @@ func (oc *DefaultNetworkController) Run(ctx context.Context) error { }() } - oc.wg.Add(1) - go func() { - defer oc.wg.Done() - oc.apbExternalRouteController.Run(1) - }() + if config.OVNKubernetesFeature.EnableMultiExternalGateway { + oc.wg.Add(1) + go func() { + defer oc.wg.Done() + oc.apbExternalRouteController.Run(1) + }() + } end := time.Since(start) klog.Infof("Completing all the Watchers took %v", end)