Add config flag 'enable-multi-external-gateway'

Signed-off-by: Peng Liu <[email protected]>

contrib/kind.sh

@@ -853,7 +853,8 @@ create_ovn_kube_manifests() {
     --multi-network-enable="${ENABLE_MULTI_NET}" \
     --ovnkube-metrics-scale-enable="${OVN_METRICS_SCALE_ENABLE}" \
     --compact-mode="${OVN_COMPACT_MODE}" \
-    --enable-interconnect="${OVN_ENABLE_INTERCONNECT}"
+    --enable-interconnect="${OVN_ENABLE_INTERCONNECT}" \
+    --enable-multi-external-gateway=true
   popd
 }
 

dist/images/daemonset.sh

@@ -300,6 +300,9 @@ while [ "$1" != "" ]; do
   --enable-interconnect)
     OVN_ENABLE_INTERCONNECT=$VALUE
     ;;
+  --enable-multi-external-gateway)
+    OVN_ENABLE_MULTI_EXTERNAL_GATEWAY=$VALUE
+    ;;
   *)
     echo "WARNING: unknown parameter \"$PARAM\""
     exit 1
@@ -456,6 +459,8 @@ ovnkube_compact_mode_enable=${COMPACT_MODE:-"false"}
 echo "ovnkube_compact_mode_enable: ${ovnkube_compact_mode_enable}"
 ovn_enable_interconnect=${OVN_ENABLE_INTERCONNECT}
 echo "ovn_enable_interconnect: ${ovn_enable_interconnect}"
+ovn_enable_multi_external_gateway=${OVN_ENABLE_MULTI_EXTERNAL_GATEWAY}
+echo "ovn_enable_multi_external_gateway: ${ovn_enable_multi_external_gateway}"
 
 ovn_image=${ovnkube_image} \
   ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
@@ -498,6 +503,7 @@ ovn_image=${ovnkube_image} \
   ovn_disable_ovn_iface_id_ver=${ovn_disable_ovn_iface_id_ver} \
   ovnkube_node_mgmt_port_netdev=${ovnkube_node_mgmt_port_netdev} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
+  ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
   ovnkube_app_name=ovnkube-node \
   j2 ../templates/ovnkube-node.yaml.j2 -o ${output_dir}/ovnkube-node.yaml
 
@@ -572,6 +578,7 @@ ovn_image=${ovnkube_image} \
   ovn_stateless_netpol_enable=${ovn_netpol_acl_enable} \
   ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
+  ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
   j2 ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml
 
 ovn_image=${ovnkube_image} \
@@ -603,6 +610,7 @@ ovn_image=${ovnkube_image} \
   ovn_gateway_mode=${ovn_gateway_mode} \
   ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
+  ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
   j2 ../templates/ovnkube-control-plane.yaml.j2 -o ${output_dir}/ovnkube-control-plane.yaml
 
 ovn_image=${image} \
@@ -683,6 +691,7 @@ ovn_image=${ovnkube_image} \
   ovn_empty_lb_events=${ovn_empty_lb_events} \
   ovn_loglevel_nb=${ovn_loglevel_nb} ovn_loglevel_sb=${ovn_loglevel_sb} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
+  ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
   j2 ../templates/ovnkube-single-node-zone.yaml.j2 -o ${output_dir}/ovnkube-single-node-zone.yaml
 
 ovn_image=${ovnkube_image} \
@@ -734,6 +743,7 @@ ovn_image=${ovnkube_image} \
   ovn_empty_lb_events=${ovn_empty_lb_events} \
   ovn_loglevel_nb=${ovn_loglevel_nb} ovn_loglevel_sb=${ovn_loglevel_sb} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
+  ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
   j2 ../templates/ovnkube-zone-controller.yaml.j2 -o ${output_dir}/ovnkube-zone-controller.yaml
 
 ovn_image=${image} \

dist/images/ovnkube.sh

@@ -84,6 +84,7 @@ fi
 # OVN_ENCAP_IP - encap IP to be used for OVN traffic on the node. mandatory in case ovnkube-node-mode=="dpu"
 # OVN_HOST_NETWORK_NAMESPACE - namespace to classify host network traffic for applying network policies
 # OVN_DISABLE_FORWARDING - disable forwarding on OVNK controlled interfaces
+# OVN_ENABLE_MULTI_EXTERNAL_GATEWAY - enable multi external gateway for ovn-kubernetes
 
 # The argument to the command is the operation to be performed
 # ovn-master ovn-controller ovn-node display display_env ovn_debug
@@ -239,6 +240,8 @@ ovn_ipfix_cache_active_timeout=${OVN_IPFIX_CACHE_ACTIVE_TIMEOUT:-} \
 ovn_stateless_netpol_enable=${OVN_STATELESS_NETPOL_ENABLE:-false}
 #OVN_ENABLE_INTERCONNECT - enable interconnect with multiple zones
 ovn_enable_interconnect=${OVN_ENABLE_INTERCONNECT:-false}
+#OVN_ENABLE_MULTI_EXTERNAL_GATEWAY - enable multi external gateway
+ovn_enable_multi_external_gateway=${OVN_ENABLE_MULTI_EXTERNAL_GATEWAY:-false}
 
 # OVNKUBE_NODE_MODE - is the mode which ovnkube node operates
 ovnkube_node_mode=${OVNKUBE_NODE_MODE:-"full"}
@@ -1125,6 +1128,12 @@ ovn-master() {
   fi
   echo "ovn_stateless_netpol_enable_flag: ${ovn_stateless_netpol_enable_flag}"
 
+  ovnkube_enable_multi_external_gateway_flag=
+  if [[ ${ovn_enable_multi_external_gateway} == "true" ]]; then
+	  ovnkube_enable_multi_external_gateway_flag="--enable-multi-external-gateway"
+  fi
+  echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}"
+
   init_node_flags=
   if [[ ${ovnkube_compact_mode_enable} == "true" ]]; then
     init_node_flags="--init-node ${K8S_NODE} --nodeport"
@@ -1165,6 +1174,7 @@ ovn-master() {
     ${ovnkube_metrics_scale_enable_flag} \
     ${multi_network_enabled_flag} \
     ${ovn_stateless_netpol_enable_flag} \
+    ${ovnkube_enable_multi_external_gateway_flag} \
     --metrics-bind-address ${ovnkube_master_metrics_bind_address} \
     --host-network-namespace ${ovn_host_network_namespace} &
 
@@ -1332,6 +1342,12 @@ ovnkube-controller() {
   fi
   echo "ovnkube_enable_interconnect_flag: ${ovnkube_enable_interconnect_flag}"
 
+  ovnkube_enable_multi_external_gateway_flag=
+  if [[ ${ovn_enable_multi_external_gateway} == "true" ]]; then
+	  ovnkube_enable_multi_external_gateway_flag="--enable-multi-external-gateway"
+  fi
+  echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}"
+
   echo "=============== ovnkube-controller ========== MASTER ONLY"
   /usr/bin/ovnkube \
     --init-ovnkube-controller ${K8S_NODE} \
@@ -1361,6 +1377,7 @@ ovnkube-controller() {
     ${ovnkube_config_duration_enable_flag} \
     ${multi_network_enabled_flag} \
     ${ovnkube_enable_interconnect_flag} \
+    ${ovnkube_enable_multi_external_gateway_flag} \
     --zone ${ovn_zone} \
     --metrics-bind-address ${ovnkube_master_metrics_bind_address} \
     --host-network-namespace ${ovn_host_network_namespace} &
@@ -1445,6 +1462,12 @@ ovn-cluster-manager() {
   fi
   echo "ovnkube_enable_interconnect_flag: ${ovnkube_enable_interconnect_flag}"
 
+  ovnkube_enable_multi_external_gateway_flag=
+  if [[ ${ovn_enable_multi_external_gateway} == "true" ]]; then
+	  ovnkube_enable_multi_external_gateway_flag="--enable-multi-external-gateway"
+  fi
+  echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}"
+
   echo "=============== ovn-cluster-manager ========== MASTER ONLY"
   /usr/bin/ovnkube \
     --init-cluster-manager ${K8S_NODE} \
@@ -1465,6 +1488,7 @@ ovn-cluster-manager() {
     ${multi_network_enabled_flag} \
     ${egressservice_enabled_flag} \
     ${ovnkube_enable_interconnect_flag} \
+    ${ovnkube_enable_multi_external_gateway_flag} \
     --metrics-bind-address ${ovnkube_cluster_manager_metrics_bind_address} \
     --host-network-namespace ${ovn_host_network_namespace} &
 
@@ -1726,6 +1750,12 @@ ovn-node() {
   ovn_zone=$(get_node_zone)
   echo "ovnkube-node's configured zone is ${ovn_zone}"
 
+  ovnkube_enable_multi_external_gateway_flag=
+  if [[ ${ovn_enable_multi_external_gateway} == "true" ]]; then
+	  ovnkube_enable_multi_external_gateway_flag="--enable-multi-external-gateway"
+  fi
+  echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}"
+
   if [[ $ovn_nbdb != "local" ]]; then
       ovn_dbs="--nb-address=${ovn_nbdb}"
   fi
@@ -1776,6 +1806,7 @@ ovn-node() {
     --metrics-bind-address ${ovnkube_node_metrics_bind_address} \
      ${ovnkube_node_mode_flag} \
     ${egress_interface} \
+    ${ovnkube_enable_multi_external_gateway_flag} \
     ${ovnkube_enable_interconnect_flag} \
     --zone ${ovn_zone} \
     --host-network-namespace ${ovn_host_network_namespace} \

dist/templates/ovnkube-control-plane.yaml.j2

@@ -160,6 +160,8 @@ spec:
               key: host_network_namespace
         - name: OVN_ENABLE_INTERCONNECT
           value: "{{ ovn_enable_interconnect }}"
+        - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY
+          value: "{{ ovn_enable_multi_external_gateway }}"
       # end of container
 
       volumes:

dist/templates/ovnkube-master.yaml.j2

@@ -279,6 +279,8 @@ spec:
           value: "{{ ovn_acl_logging_rate_limit }}"
         - name: OVN_STATELESS_NETPOL_ENABLE
           value: "{{ ovn_stateless_netpol_enable }}"
+        - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY
+          value: "{{ ovn_enable_multi_external_gateway }}"
         - name: OVN_HOST_NETWORK_NAMESPACE
           valueFrom:
             configMapKeyRef:

dist/templates/ovnkube-node.yaml.j2

@@ -219,6 +219,8 @@ spec:
           value: "{{ ovn_multi_network_enable }}"
         - name: OVN_ENABLE_INTERCONNECT
           value: "{{ ovn_enable_interconnect }}"
+        - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY
+          value: "{{ ovn_enable_multi_external_gateway }}"
         {% endif -%}
         {% if ovnkube_app_name=="ovnkube-node-dpu-host" -%}
         - name: OVNKUBE_NODE_MODE

dist/templates/ovnkube-single-node-zone.yaml.j2

@@ -340,6 +340,8 @@ spec:
           value: "{{ ovn_acl_logging_rate_limit }}"
         - name: OVN_ENABLE_INTERCONNECT
           value: "{{ ovn_enable_interconnect }}"
+        - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY
+          value: "{{ ovn_enable_multi_external_gateway }}"
         - name: OVN_HOST_NETWORK_NAMESPACE
           valueFrom:
             configMapKeyRef:
@@ -524,6 +526,8 @@ spec:
           value: "local"
         - name: OVN_ENABLE_INTERCONNECT
           value: "{{ ovn_enable_interconnect }}"
+        - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY
+          value: "{{ ovn_enable_multi_external_gateway }}"
 
         readinessProbe:
           exec:

dist/templates/ovnkube-zone-controller.yaml.j2

@@ -356,6 +356,8 @@ spec:
           value: "{{ ovn_acl_logging_rate_limit }}"
         - name: OVN_ENABLE_INTERCONNECT
           value: "{{ ovn_enable_interconnect }}"
+        - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY
+          value: "{{ ovn_enable_multi_external_gateway }}"
         - name: OVN_HOST_NETWORK_NAMESPACE
           valueFrom:
             configMapKeyRef:

go-controller/pkg/config/config.go

@@ -359,6 +359,7 @@ type OVNKubernetesFeatureConfig struct {
 	EnableMultiNetworkPolicy        bool `gcfg:"enable-multi-networkpolicy"`
 	EnableStatelessNetPol           bool `gcfg:"enable-stateless-netpol"`
 	EnableInterconnect              bool `gcfg:"enable-interconnect"`
+	EnableMultiExternalGateway      bool `gcfg:"enable-multi-external-gateway"`
 }
 
 // GatewayMode holds the node gateway mode
@@ -971,6 +972,12 @@ var OVNK8sFeatureFlags = []cli.Flag{
 		Destination: &cliConfig.OVNKubernetesFeature.EnableEgressService,
 		Value:       OVNKubernetesFeature.EnableEgressService,
 	},
+	&cli.BoolFlag{
+		Name:        "enable-multi-external-gateway",
+		Usage:       "Configure to use AdminPolicyBasedExternalRoute CRD feature with ovn-kubernetes.",
+		Destination: &cliConfig.OVNKubernetesFeature.EnableMultiExternalGateway,
+		Value:       OVNKubernetesFeature.EnableMultiExternalGateway,
+	},
 }
 
 // K8sFlags capture Kubernetes-related options

go-controller/pkg/config/config_test.go

@@ -219,6 +219,7 @@ egressip-node-healthcheck-port=1234
 enable-multi-network=false
 enable-multi-networkpolicy=false
 enable-interconnect=false
+enable-multi-external-gateway=false
 `
 
 	var newData string
@@ -317,6 +318,7 @@ var _ = Describe("Config Operations", func() {
 			gomega.Expect(OVNKubernetesFeature.EnableMultiNetwork).To(gomega.BeFalse())
 			gomega.Expect(OVNKubernetesFeature.EnableMultiNetworkPolicy).To(gomega.BeFalse())
 			gomega.Expect(OVNKubernetesFeature.EnableInterconnect).To(gomega.BeFalse())
+			gomega.Expect(OVNKubernetesFeature.EnableMultiExternalGateway).To(gomega.BeFalse())
 
 			for _, a := range []OvnAuthConfig{OvnNorth, OvnSouth} {
 				gomega.Expect(a.Scheme).To(gomega.Equal(OvnDBSchemeUnix))
@@ -555,6 +557,7 @@ var _ = Describe("Config Operations", func() {
 			"enable-multi-network=true",
 			"enable-multi-networkpolicy=true",
 			"enable-interconnect=true",
+			"enable-multi-external-gateway=true",
 			"zone=foo",
 		)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -634,6 +637,7 @@ var _ = Describe("Config Operations", func() {
 			gomega.Expect(OVNKubernetesFeature.EgressIPNodeHealthCheckPort).To(gomega.Equal(1234))
 			gomega.Expect(OVNKubernetesFeature.EnableMultiNetwork).To(gomega.BeTrue())
 			gomega.Expect(OVNKubernetesFeature.EnableInterconnect).To(gomega.BeTrue())
+			gomega.Expect(OVNKubernetesFeature.EnableMultiExternalGateway).To(gomega.BeTrue())
 			gomega.Expect(HybridOverlay.ClusterSubnets).To(gomega.Equal([]CIDRNetworkEntry{
 				{ovntest.MustParseIPNet("11.132.0.0/14"), 23},
 			}))
@@ -724,6 +728,7 @@ var _ = Describe("Config Operations", func() {
 			gomega.Expect(OVNKubernetesFeature.EnableMultiNetwork).To(gomega.BeTrue())
 			gomega.Expect(OVNKubernetesFeature.EnableMultiNetworkPolicy).To(gomega.BeTrue())
 			gomega.Expect(OVNKubernetesFeature.EnableInterconnect).To(gomega.BeTrue())
+			gomega.Expect(OVNKubernetesFeature.EnableMultiExternalGateway).To(gomega.BeTrue())
 			gomega.Expect(HybridOverlay.ClusterSubnets).To(gomega.Equal([]CIDRNetworkEntry{
 				{ovntest.MustParseIPNet("11.132.0.0/14"), 23},
 			}))
@@ -786,6 +791,7 @@ var _ = Describe("Config Operations", func() {
 			"-enable-multi-network=true",
 			"-enable-multi-networkpolicy=true",
 			"-enable-interconnect=true",
+			"-enable-multi-external-gateway=true",
 			"-healthz-bind-address=0.0.0.0:4321",
 			"-zone=bar",
 		}

go-controller/pkg/node/default_node_network_controller.go

@@ -136,15 +136,6 @@ func NewDefaultNodeNetworkController(cnnci *CommonNodeNetworkControllerInfo) (*D
 		}
 	}
 
-	nc.apbExternalRouteNodeController, err = apbroute.NewExternalNodeController(
-		cnnci.apbExternalRouteClient,
-		nc.watchFactory.PodCoreInformer(),
-		nc.watchFactory.NamespaceInformer(),
-		stopChan)
-	if err != nil {
-		return nil, err
-	}
-
 	nc.initRetryFrameworkForNode()
 
 	return nc, nil
@@ -936,11 +927,22 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error {
 			c.Run(1)
 		}()
 	}
-	nc.wg.Add(1)
-	go func() {
-		defer nc.wg.Done()
-		nc.apbExternalRouteNodeController.Run(1)
-	}()
+	if config.OVNKubernetesFeature.EnableMultiExternalGateway {
+		nc.apbExternalRouteNodeController, err = apbroute.NewExternalNodeController(
+			nc.apbExternalRouteClient,
+			nc.watchFactory.PodCoreInformer(),
+			nc.watchFactory.NamespaceInformer(),
+			nc.stopChan)
+		if err != nil {
+			return err
+		}
+
+		nc.wg.Add(1)
+		go func() {
+			defer nc.wg.Done()
+			nc.apbExternalRouteNodeController.Run(1)
+		}()
+	}
 
 	nc.wg.Add(1)
 	go func() {
@@ -1073,6 +1075,9 @@ func (nc *DefaultNodeNetworkController) checkAndDeleteStaleConntrackEntries() {
 }
 
 func (nc *DefaultNodeNetworkController) syncConntrackForExternalGateways(newNs *kapi.Namespace) error {
+	if nc.apbExternalRouteNodeController == nil {
+		return nil
+	}
 	gatewayIPs, err := nc.apbExternalRouteNodeController.GetAdminPolicyBasedExternalRouteIPsForTargetNamespace(newNs.Name)
 	if err != nil {
 		return fmt.Errorf("unable to retrieve gateway IPs for Admin Policy Based External Route objects: %w", err)

go-controller/pkg/ovn/default_network_controller.go

@@ -514,11 +514,13 @@ func (oc *DefaultNetworkController) Run(ctx context.Context) error {
 		}()
 	}
 
-	oc.wg.Add(1)
-	go func() {
-		defer oc.wg.Done()
-		oc.apbExternalRouteController.Run(1)
-	}()
+	if config.OVNKubernetesFeature.EnableMultiExternalGateway {
+		oc.wg.Add(1)
+		go func() {
+			defer oc.wg.Done()
+			oc.apbExternalRouteController.Run(1)
+		}()
+	}
 
 	end := time.Since(start)
 	klog.Infof("Completing all the Watchers took %v", end)