StatusManager: consolidate status updates from different zones

dist/images/ovnkube.sh

@@ -1952,6 +1952,12 @@ ovn-cluster-manager() {
   fi
   echo "egressservice_enabled_flag=${egressservice_enabled_flag}"
 
+  egressfirewall_enabled_flag=
+  if [[ ${ovn_egressfirewall_enable} == "true" ]]; then
+	  egressfirewall_enabled_flag="--enable-egress-firewall"
+  fi
+  echo "egressfirewall_enabled_flag=${egressfirewall_enabled_flag}"
+
   hybrid_overlay_flags=
   if [[ ${ovn_hybrid_overlay_enable} == "true" ]]; then
     hybrid_overlay_flags="--enable-hybrid-overlay"
@@ -2029,6 +2035,7 @@ ovn-cluster-manager() {
 
   echo "=============== ovn-cluster-manager ========== MASTER ONLY"
   /usr/bin/ovnkube --init-cluster-manager ${K8S_NODE} \
+    ${egressfirewall_enabled_flag} \
     ${egressip_enabled_flag} \
     ${egressip_healthcheck_port_flag} \
     ${egressservice_enabled_flag} \

dist/templates/k8s.ovn.org_adminpolicybasedexternalroutes.yaml.j2

@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.10.0
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: adminpolicybasedexternalroutes.k8s.ovn.org
 spec:
   group: k8s.ovn.org
@@ -273,14 +272,14 @@ spec:
                 items:
                   type: string
                 type: array
+                x-kubernetes-list-type: set
               status:
                 description: A concise indication of whether the AdminPolicyBasedRoute
                   resource is applied with success
                 type: string
             required:
             - lastTransitionTime
             - messages
-            - status
             type: object
         required:
         - spec

dist/templates/k8s.ovn.org_egressfirewalls.yaml.j2

@@ -156,12 +156,20 @@ spec:
           status:
             description: Observed status of EgressFirewall
             properties:
+              messages:
+                items:
+                  type: string
+                type: array
+                x-kubernetes-list-type: set
               status:
                 type: string
+            required:
+            - messages
             type: object
         required:
         - spec
         type: object
     served: true
     storage: true
-    subresources: {}
+    subresources:
+      status: {}

dist/templates/k8s.ovn.org_egressservices.yaml.j2

@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.3
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: egressservices.k8s.ovn.org
 spec:
   group: k8s.ovn.org

dist/templates/rbac-ovnkube-cluster-manager.yaml.j2

@@ -60,6 +60,8 @@ rules:
       resources:
           - egressips
           - egressservices
+          - adminpolicybasedexternalroutes
+          - egressfirewalls
       verbs: [ "get", "list", "watch" ]
     - apiGroups: ["k8s.ovn.org"]
       resources:
@@ -76,3 +78,8 @@ rules:
           - nodes/status
           - services/status
       verbs: [ "patch", "update" ]
+    - apiGroups: ["k8s.ovn.org"]
+      resources:
+        - adminpolicybasedexternalroutes/status
+        - egressfirewalls/status
+      verbs: [ "patch", "update" ]

dist/templates/rbac-ovnkube-master.yaml.j2

@@ -96,7 +96,7 @@ rules:
       verbs: [ "patch", "update" ]
     - apiGroups: ["k8s.ovn.org"]
       resources:
-          - egressfirewalls
+          - egressfirewalls/status
           - egressips
           - egressqoses
           - egressservices/status

dist/templates/rbac-ovnkube-node.yaml.j2

@@ -151,7 +151,7 @@ rules:
       verbs: ["list", "get", "watch"]
     - apiGroups: ["k8s.ovn.org"]
       resources:
-          - egressfirewalls
+          - egressfirewalls/status
           - adminpolicybasedexternalroutes/status
       verbs: [ "patch", "update" ]
     - apiGroups: ["policy.networking.k8s.io"]

go-controller/go.mod

@@ -58,6 +58,7 @@ require (
 	kubevirt.io/api v1.0.0-alpha.0
 	sigs.k8s.io/controller-runtime v0.15.1
 	sigs.k8s.io/network-policy-api v0.1.2
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3
 )
 
 require (
@@ -124,7 +125,6 @@ require (
 	kubevirt.io/containerized-data-importer-api v1.55.0 // indirect
 	kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 

go-controller/hack/update-codegen.sh

@@ -9,16 +9,14 @@ if [ -z "${crds}" ]; then
   exit
 fi
 
-if  ! ( command -v controller-gen > /dev/null ); then
-  echo "controller-gen not found, installing sigs.k8s.io/controller-tools"
-  olddir="${PWD}"
-  builddir="$(mktemp -d)"
-  cd "${builddir}"
-  GO111MODULE=on go get -u sigs.k8s.io/controller-tools/cmd/controller-gen # currently on v0.13.0
-  cd "${olddir}"
-  if [[ "${builddir}" == /tmp/* ]]; then #paranoia
-      rm -rf "${builddir}"
-  fi
+SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
+olddir="${PWD}"
+builddir="$(mktemp -d)"
+cd "${builddir}"
+GO111MODULE=on go install sigs.k8s.io/controller-tools/cmd/controller-gen@latest
+cd "${olddir}"
+if [[ "${builddir}" == /tmp/* ]]; then #paranoia
+    rm -rf "${builddir}"
 fi
 
 for crd in ${crds}; do
@@ -29,6 +27,12 @@ for crd in ${crds}; do
     -O zz_generated.deepcopy \
     --bounding-dirs github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd
 
+  echo "Generating apply configuration for $crd"
+  applyconfiguration-gen \
+    --go-header-file hack/boilerplate.go.txt \
+    --input-dirs github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1 \
+    --output-package github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1/apis/applyconfiguration \
+    "$@"
 
   echo "Generating clientset for $crd"
   client-gen \
@@ -37,6 +41,7 @@ for crd in ${crds}; do
     --input-base "" \
     --input github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1 \
     --output-package github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1/apis/clientset \
+    --apply-configuration-package github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1/apis/applyconfiguration \
     --plural-exceptions="EgressQoS:EgressQoSes" \
     "$@"
 
@@ -57,6 +62,11 @@ for crd in ${crds}; do
     --output-package github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1/apis/informers \
     --plural-exceptions="EgressQoS:EgressQoSes" \
     "$@"
+
+  echo "Copying apis for $crd"
+  rm -rf $SCRIPT_ROOT/pkg/crd/$crd/v1/apis
+  cp -r github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/$crd/v1/apis $SCRIPT_ROOT/pkg/crd/$crd/v1
+
 done
 
 echo "Generating CRDs"
@@ -86,3 +96,7 @@ echo "Copying Admin Network Policy CRD"
 curl -sSL https://raw.githubusercontent.com/kubernetes-sigs/network-policy-api/v0.1.2/config/crd/experimental/policy.networking.k8s.io_adminnetworkpolicies.yaml -o ../dist/templates/policy.networking.k8s.io_adminnetworkpolicies.yaml
 echo "Copying Baseline Admin Network Policy CRD"
 curl -sSL https://raw.githubusercontent.com/kubernetes-sigs/network-policy-api/v0.1.2/config/crd/experimental/policy.networking.k8s.io_baselineadminnetworkpolicies.yaml -o ../dist/templates/policy.networking.k8s.io_baselineadminnetworkpolicies.yaml
+echo "Copying adminpolicybasedexternalroutes CRD"
+cp _output/crds/k8s.ovn.org_adminpolicybasedexternalroutes.yaml ../dist/templates/k8s.ovn.org_adminpolicybasedexternalroutes.yaml.j2
+echo "Copying egressService CRD"
+cp _output/crds/k8s.ovn.org_egressservices.yaml ../dist/templates/k8s.ovn.org_egressservices.yaml.j2

go-controller/pkg/clustermanager/clustermanager.go

@@ -7,6 +7,7 @@ import (
 	"sync"
 
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/egressservice"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/clustermanager/status_manager"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/unidling"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/healthcheck"
@@ -32,7 +33,6 @@ type ClusterManager struct {
 	defaultNetClusterController *networkClusterController
 	zoneClusterController       *zoneClusterController
 	wf                          *factory.WatchFactory
-	wg                          *sync.WaitGroup
 	secondaryNetClusterManager  *secondaryNetworkClusterManager
 	// Controller used for programming node allocation for egress IP
 	// The OVN DB setup is handled by egressIPZoneController that runs in ovnkube-controller
@@ -43,7 +43,8 @@ type ClusterManager struct {
 
 	// unique identity for clusterManager running on different ovnkube-cluster-manager instance,
 	// used for leader election
-	identity string
+	identity      string
+	statusManager *status_manager.StatusManager
 }
 
 // NewClusterManager creates a new cluster manager to manage the cluster nodes.
@@ -61,10 +62,10 @@ func NewClusterManager(ovnClient *util.OVNClusterManagerClientset, wf *factory.W
 		client:                      ovnClient.KubeClient,
 		defaultNetClusterController: defaultNetClusterController,
 		zoneClusterController:       zoneClusterController,
-		wg:                          wg,
 		wf:                          wf,
 		recorder:                    recorder,
 		identity:                    identity,
+		statusManager:               status_manager.NewStatusManager(wf, ovnClient),
 	}
 
 	if config.OVNKubernetesFeature.EnableMultiNetwork {
@@ -145,6 +146,11 @@ func (cm *ClusterManager) Start(ctx context.Context) error {
 			return err
 		}
 	}
+
+	if err := cm.statusManager.Start(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -162,4 +168,5 @@ func (cm *ClusterManager) Stop() {
 	if config.OVNKubernetesFeature.EnableEgressService {
 		cm.egressServiceController.Stop()
 	}
+	cm.statusManager.Stop()
 }

go-controller/pkg/clustermanager/status_manager/apbroute_manager.go

@@ -0,0 +1,79 @@
+package status_manager
+
+import (
+	"context"
+	"strings"
+
+	adminpolicybasedrouteapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1"
+	adminpolicybasedrouteapply "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/applyconfiguration/adminpolicybasedroute/v1"
+	adminpolicybasedrouteclientset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/clientset/versioned"
+	adminpolicybasedroutelisters "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/listers/adminpolicybasedroute/v1"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type apbRouteManager struct {
+	lister adminpolicybasedroutelisters.AdminPolicyBasedExternalRouteLister
+	client adminpolicybasedrouteclientset.Interface
+}
+
+func newAPBRouteManager(lister adminpolicybasedroutelisters.AdminPolicyBasedExternalRouteLister, client adminpolicybasedrouteclientset.Interface) *apbRouteManager {
+	return &apbRouteManager{
+		lister: lister,
+		client: client,
+	}
+}
+
+//lint:ignore U1000 generic interfaces throw false-positives https://github.com/dominikh/go-tools/issues/1440
+func (m *apbRouteManager) get(namespace, name string) (*adminpolicybasedrouteapi.AdminPolicyBasedExternalRoute, error) {
+	return m.lister.Get(name)
+}
+
+//lint:ignore U1000 generic interfaces throw false-positives
+func (m *apbRouteManager) getMessages(route *adminpolicybasedrouteapi.AdminPolicyBasedExternalRoute) []string {
+	return route.Status.Messages
+}
+
+//lint:ignore U1000 generic interfaces throw false-positives
+func (m *apbRouteManager) updateStatus(route *adminpolicybasedrouteapi.AdminPolicyBasedExternalRoute, applyOpts *metav1.ApplyOptions,
+	applyEmptyOrFailed bool) error {
+	if route == nil {
+		return nil
+	}
+	newStatus := adminpolicybasedrouteapi.SuccessStatus
+	for _, message := range route.Status.Messages {
+		if strings.Contains(message, types.APBRouteErrorMsg) {
+			newStatus = adminpolicybasedrouteapi.FailStatus
+			break
+		}
+	}
+	if applyEmptyOrFailed && newStatus != adminpolicybasedrouteapi.FailStatus {
+		newStatus = ""
+	}
+
+	if route.Status.Status == newStatus {
+		// already set to the same value
+		return nil
+	}
+
+	applyStatus := adminpolicybasedrouteapply.AdminPolicyBasedRouteStatus()
+
+	if newStatus != "" {
+		applyStatus.WithStatus(newStatus)
+	}
+
+	applyObj := adminpolicybasedrouteapply.AdminPolicyBasedExternalRoute(route.Name).
+		WithStatus(applyStatus)
+
+	_, err := m.client.K8sV1().AdminPolicyBasedExternalRoutes().ApplyStatus(context.TODO(), applyObj, *applyOpts)
+	return err
+}
+
+//lint:ignore U1000 generic interfaces throw false-positives
+func (m *apbRouteManager) cleanupStatus(route *adminpolicybasedrouteapi.AdminPolicyBasedExternalRoute, applyOpts *metav1.ApplyOptions) error {
+	applyObj := adminpolicybasedrouteapply.AdminPolicyBasedExternalRoute(route.Name).
+		WithStatus(adminpolicybasedrouteapply.AdminPolicyBasedRouteStatus())
+	_, err := m.client.K8sV1().AdminPolicyBasedExternalRoutes().ApplyStatus(context.TODO(), applyObj, *applyOpts)
+	return err
+}