CNF-14426: Add Interfaces Handling to Hardware Provisioning (#201)

Signed-off-by: Tao Liu <[email protected]>

api/hardwaremanagement/v1alpha1/node.go

@@ -18,6 +18,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+type Interface struct {
+	Name  string `json:"name"`  // The name of the network interface (e.g., eth0, ens33)
+	Label string `json:"label"` // The label of the interface
+	// +kubebuilder:validation:Pattern=`^([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2})$`
+	MACAddress string `json:"macAddress"` // The MAC address of the interface
+}
+
 // NodeSpec describes a node presents a hardware server
 type NodeSpec struct {
 	// NodePool
@@ -45,7 +52,7 @@ type BMC struct {
 type NodeStatus struct {
 	BMC *BMC `json:"bmc,omitempty"`
 
-	BootMACAddress string `json:"bootMACAddress,omitempty"`
+	Interfaces []*Interface `json:"interfaces,omitempty"`
 
 	Hostname string `json:"hostname,omitempty"`
 

api/hardwaremanagement/v1alpha1/node_pools.go

@@ -49,6 +49,8 @@ type NodeGroup struct {
 	Name      string `json:"name" yaml:"name"`
 	HwProfile string `json:"hwProfile"`
 	Size      int    `json:"size" yaml:"size"`
+	// +kubebuilder:validation:MinItems=1
+	Interfaces []string `json:"interfaces,omitempty"`
 }
 
 type Properties struct {

config/crd/bases/o2ims-hardwaremanagement.oran.openshift.io_nodepools.yaml

@@ -55,6 +55,11 @@ spec:
                   properties:
                     hwProfile:
                       type: string
+                    interfaces:
+                      items:
+                        type: string
+                      minItems: 1
+                      type: array
                     name:
                       type: string
                     size:

config/crd/bases/o2ims-hardwaremanagement.oran.openshift.io_nodes.yaml

@@ -70,8 +70,6 @@ spec:
                       should contain the keys `username` and `password`.
                     type: string
                 type: object
-              bootMACAddress:
-                type: string
               conditions:
                 description: |-
                   Conditions represent the observations of the NodeStatus's current state.
@@ -146,6 +144,22 @@ spec:
                 type: array
               hostname:
                 type: string
+              interfaces:
+                items:
+                  properties:
+                    label:
+                      type: string
+                    macAddress:
+                      pattern: ^([0-9A-Fa-f]{2}[:]){5}([0-9A-Fa-f]{2})$
+                      type: string
+                    name:
+                      type: string
+                  required:
+                  - label
+                  - macAddress
+                  - name
+                  type: object
+                type: array
             type: object
         type: object
     served: true

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.14.0
 	github.com/onsi/gomega v1.30.0
 	github.com/openshift-kni/oran-o2ims/api/hardwaremanagement v0.0.0-00010101000000-000000000000
+	github.com/openshift/assisted-service/api v0.0.0-20240405132132-484ec5c683c6
 	github.com/peterhellberg/link v1.2.0
 	github.com/prometheus/client_golang v1.18.0
 	github.com/spf13/cobra v1.7.0
@@ -86,7 +87,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/openshift/api v0.0.0-20240124164020-e2ce40831f2e // indirect
-	github.com/openshift/assisted-service/api v0.0.0-20240405132132-484ec5c683c6 // indirect
 	github.com/openshift/assisted-service/models v0.0.0 // indirect
 	github.com/openshift/custom-resource-status v1.1.3-0.20220503160415-f2fdb4999d87 // indirect
 	github.com/openshift/hive/apis v0.0.0-20240306163002-9c5806a63531 // indirect

internal/controllers/clusterrequest_controller.go

@@ -74,6 +74,7 @@ type nodeInfo struct {
 	bmcAddress     string
 	bmcCredentials string
 	nodeName       string
+	interfaces     []*hwv1alpha1.Interface
 }
 
 type deleteOrUpdateEvent interface {
@@ -629,10 +630,11 @@ func (t *clusterRequestReconcilerTask) handleRenderHardwareTemplate(ctx context.
 			"failed to get the Hardware template from ConfigMap %s, err: %w", hwTemplateCmName, err)
 	}
 
-	// count the nodes per group
 	roleCounts := make(map[string]int)
-	for _, node := range clusterInstance.Spec.Nodes {
-		roleCounts[node.Role]++
+	err = utils.ProcessClusterNodeGroups(clusterInstance, nodeGroup, roleCounts)
+	if err != nil {
+		return nil, fmt.Errorf(
+			"failed to get the process node spec err: %w", err)
 	}
 
 	for i, group := range nodeGroup {
@@ -647,6 +649,11 @@ func (t *clusterRequestReconcilerTask) handleRenderHardwareTemplate(ctx context.
 	nodePool.ObjectMeta.Name = clusterInstance.GetName()
 	nodePool.ObjectMeta.Namespace = hwTemplateCm.Data[utils.HwTemplatePluginMgr]
 
+	// Add boot interface label to the generated nodePool
+	annotation := make(map[string]string)
+	annotation[utils.HwTemplateBootIfaceLabel] = hwTemplateCm.Data[utils.HwTemplateBootIfaceLabel]
+	nodePool.SetAnnotations(annotation)
+
 	// Add ClusterRequest labels to the generated nodePool
 	labels := make(map[string]string)
 	labels[clusterRequestNameLabel] = t.object.Name
@@ -1552,7 +1559,7 @@ func (t *clusterRequestReconcilerTask) checkNodePoolProvisionStatus(ctx context.
 func (t *clusterRequestReconcilerTask) updateClusterInstance(ctx context.Context,
 	clusterInstance *siteconfig.ClusterInstance, nodePool *hwv1alpha1.NodePool) bool {
 
-	hwNodes, macAddresses := t.collectNodeDetails(ctx, nodePool)
+	hwNodes := t.collectNodeDetails(ctx, nodePool)
 	if hwNodes == nil {
 		return false
 	}
@@ -1561,7 +1568,7 @@ func (t *clusterRequestReconcilerTask) updateClusterInstance(ctx context.Context
 		return false
 	}
 
-	if !t.applyNodeConfiguration(ctx, hwNodes, macAddresses, nodePool, clusterInstance) {
+	if !t.applyNodeConfiguration(ctx, hwNodes, nodePool, clusterInstance) {
 		return false
 	}
 
@@ -1588,13 +1595,12 @@ func (t *clusterRequestReconcilerTask) waitForHardwareData(ctx context.Context,
 	return false
 }
 
-// collectNodeDetails collects BMC details and boot MAC addresses.
+// collectNodeDetails collects BMC and node interfaces details
 func (t *clusterRequestReconcilerTask) collectNodeDetails(ctx context.Context,
-	nodePool *hwv1alpha1.NodePool) (map[string][]nodeInfo, map[string]string) {
+	nodePool *hwv1alpha1.NodePool) map[string][]nodeInfo {
 
 	// hwNodes maps a group name to a slice of NodeInfo
 	hwNodes := make(map[string][]nodeInfo)
-	macAddresses := make(map[string]string)
 
 	for _, nodeName := range nodePool.Status.Properties.NodeNames {
 		node := &hwv1alpha1.Node{}
@@ -1608,7 +1614,7 @@ func (t *clusterRequestReconcilerTask) collectNodeDetails(ctx context.Context,
 				slog.String("error", err.Error()),
 				slog.Bool("exists", exists),
 			)
-			return nil, nil
+			return nil
 		}
 		if !exists {
 			t.logger.ErrorContext(
@@ -1618,7 +1624,7 @@ func (t *clusterRequestReconcilerTask) collectNodeDetails(ctx context.Context,
 				slog.String("namespace", nodePool.Namespace),
 				slog.Bool("exists", exists),
 			)
-			return nil, nil
+			return nil
 		}
 		// Verify the node object is generated from the expected pool
 		if node.Spec.NodePool != nodePool.GetName() {
@@ -1628,7 +1634,7 @@ func (t *clusterRequestReconcilerTask) collectNodeDetails(ctx context.Context,
 				slog.String("name", node.GetName()),
 				slog.String("pool", nodePool.GetName()),
 			)
-			return nil, nil
+			return nil
 		}
 
 		if node.Status.BMC == nil {
@@ -1638,21 +1644,18 @@ func (t *clusterRequestReconcilerTask) collectNodeDetails(ctx context.Context,
 				slog.String("name", node.GetName()),
 				slog.String("pool", nodePool.GetName()),
 			)
-			return nil, nil
+			return nil
 		}
 		// Store the nodeInfo per group
 		hwNodes[node.Spec.GroupName] = append(hwNodes[node.Spec.GroupName], nodeInfo{
 			bmcAddress:     node.Status.BMC.Address,
 			bmcCredentials: node.Status.BMC.CredentialsName,
 			nodeName:       node.Name,
+			interfaces:     node.Status.Interfaces,
 		})
-
-		if node.Status.BootMACAddress != "" {
-			macAddresses[node.Status.BMC.Address] = node.Status.BootMACAddress
-		}
 	}
 
-	return hwNodes, macAddresses
+	return hwNodes
 }
 
 // copyBMCSecrets copies BMC secrets from the plugin namespace to the cluster namespace.
@@ -1677,9 +1680,9 @@ func (t *clusterRequestReconcilerTask) copyBMCSecrets(ctx context.Context, hwNod
 	return true
 }
 
-// applyNodeConfiguration updates the clusterInstance with BMC details and bootMacAddress.
+// applyNodeConfiguration updates the clusterInstance with BMC details, interface MACAddress and bootMACAddress
 func (t *clusterRequestReconcilerTask) applyNodeConfiguration(ctx context.Context, hwNodes map[string][]nodeInfo,
-	macAddresses map[string]string, nodePool *hwv1alpha1.NodePool, clusterInstance *siteconfig.ClusterInstance) bool {
+	nodePool *hwv1alpha1.NodePool, clusterInstance *siteconfig.ClusterInstance) bool {
 
 	for i, node := range clusterInstance.Spec.Nodes {
 		// Check if the node's role matches any key in hwNodes
@@ -1690,10 +1693,29 @@ func (t *clusterRequestReconcilerTask) applyNodeConfiguration(ctx context.Contex
 
 		clusterInstance.Spec.Nodes[i].BmcAddress = nodeInfos[0].bmcAddress
 		clusterInstance.Spec.Nodes[i].BmcCredentialsName = siteconfig.BmcCredentialsName{Name: nodeInfos[0].bmcCredentials}
-		if mac, macExists := macAddresses[nodeInfos[0].bmcAddress]; macExists {
-			clusterInstance.Spec.Nodes[i].BootMACAddress = mac
+		// Get the boot MAC address based on the interface label
+		bootMAC, err := utils.GetBootMacAddress(nodeInfos[0].interfaces, nodePool)
+		if err != nil {
+			t.logger.ErrorContext(
+				ctx,
+				"Fail to get the node boot MAC address",
+				slog.String("name", node.HostName),
+				slog.String("error", err.Error()),
+			)
+			return false
 		}
-		// indicates which host has been assigned to the node
+		clusterInstance.Spec.Nodes[i].BootMACAddress = bootMAC
+
+		// Populate the MAC address for each interface
+		for j, iface := range clusterInstance.Spec.Nodes[i].NodeNetwork.Interfaces {
+			for _, nodeIface := range nodeInfos[0].interfaces {
+				if nodeIface.Name == iface.Name {
+					clusterInstance.Spec.Nodes[i].NodeNetwork.Interfaces[j].MacAddress = nodeIface.MACAddress
+				}
+			}
+		}
+
+		// Indicates which host has been assigned to the node
 		if !t.updateNodeStatusWithHostname(ctx, nodeInfos[0].nodeName, node.HostName,
 			nodePool.Namespace) {
 			return false