Implement multi-NIC-same-VPC validation logic to validate that secondary NICs are in the same VPC as the primary NIC.

Author: tyuchn

cmd/csi_driver/main.go

@@ -77,9 +77,6 @@ func main() {
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	netlinker := network.NewNetlink()
-	nodeClient := network.NewK8sClient()
-	networkIntf := network.Manager(netlinker, nodeClient)
 
 	config := &driver.LustreDriverConfig{
 		Name:                   driver.DefaultName,
@@ -102,8 +99,12 @@ func main() {
 		klog.Infof("Metadata service setup: %+v", meta)
 		config.MetadataService = meta
 
+		netlinker := network.NewNetlink()
+		nodeClient := network.NewK8sClient()
+		networkIntf := network.Manager(netlinker, nodeClient, meta)
+
 		config.Mounter = mount.New("")
-		nics, err := networkIntf.GetGvnicNames()
+		nics, err := networkIntf.GetStandardNICs()
 		if err != nil {
 			klog.Fatalf("Error getting nic names: %v", err)
 		}

pkg/cloud_provider/metadata/fake.go

@@ -16,7 +16,9 @@ limitations under the License.
 
 package metadata
 
-type fakeServiceManager struct{}
+type fakeServiceManager struct {
+	GetNetworkInterfacesHook func() ([]NetworkInterface, error)
+}
 
 var _ Service = &fakeServiceManager{}
 
@@ -31,3 +33,10 @@ func (manager *fakeServiceManager) GetZone() string {
 func (manager *fakeServiceManager) GetProject() string {
 	return "test-project"
 }
+
+func (manager *fakeServiceManager) GetNetworkInterfaces() ([]NetworkInterface, error) {
+	if manager.GetNetworkInterfacesHook != nil {
+		return manager.GetNetworkInterfacesHook()
+	}
+	return nil, nil
+}

pkg/cloud_provider/metadata/metadata.go

@@ -18,20 +18,28 @@ package metadata
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	"cloud.google.com/go/compute/metadata"
 )
 
+type NetworkInterface struct {
+	Network string `json:"network"`
+	Mac     string `json:"mac"`
+}
+
 type Service interface {
 	GetZone() string
 	GetProject() string
+	GetNetworkInterfaces() ([]NetworkInterface, error)
 }
 
 type metadataServiceManager struct {
 	// Current zone the driver is running in.
 	zone    string
 	project string
+	nics    []NetworkInterface
 }
 
 var _ Service = &metadataServiceManager{}
@@ -46,9 +54,21 @@ func NewMetadataService(ctx context.Context) (Service, error) {
 		return nil, fmt.Errorf("failed to get project: %w", err)
 	}
 
+	// Fetch network interfaces recursively
+	nicsJSON, err := metadata.GetWithContext(ctx, "instance/network-interfaces/?recursive=true")
+	if err != nil {
+		return nil, fmt.Errorf("failed to get network interfaces: %w", err)
+	}
+
+	var nics []NetworkInterface
+	if err := json.Unmarshal([]byte(nicsJSON), &nics); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal network interfaces: %w", err)
+	}
+
 	return &metadataServiceManager{
-		project: projectID,
 		zone:    zone,
+		project: projectID,
+		nics:    nics,
 	}, nil
 }
 
@@ -59,3 +79,7 @@ func (manager *metadataServiceManager) GetZone() string {
 func (manager *metadataServiceManager) GetProject() string {
 	return manager.project
 }
+
+func (manager *metadataServiceManager) GetNetworkInterfaces() ([]NetworkInterface, error) {
+	return manager.nics, nil
+}

pkg/csi_driver/node.go

@@ -159,7 +159,7 @@ func (s *nodeServer) NodeStageVolume(_ context.Context, req *csi.NodeStageVolume
 	if !s.driver.config.DisableMultiNIC {
 		netlinker := network.NewNetlink()
 		nodeClient := network.NewK8sClient()
-		networkIntf := network.Manager(netlinker, nodeClient)
+		networkIntf := network.Manager(netlinker, nodeClient, s.driver.config.MetadataService)
 		klog.V(4).Infof("Multi Nic feature is enabled and will configure route for Lustre instance: %v, IP: %v", volumeID, source)
 		nics := s.driver.config.AdditionalNics
 		for _, nicName := range nics {

pkg/network/network.go

@@ -22,7 +22,9 @@ import (
 	"net"
 	"os"
 	"strconv"
+	"strings"
 
+	"github.com/GoogleCloudPlatform/lustre-csi-driver/pkg/cloud_provider/metadata"
 	"github.com/GoogleCloudPlatform/lustre-csi-driver/pkg/k8sclient"
 	"github.com/safchain/ethtool"
 	"github.com/vishvananda/netlink"
@@ -47,17 +49,23 @@ type netlinker interface {
 	RuleAdd(rule *netlink.Rule) error
 	RuleList(family int) ([]netlink.Rule, error)
 	RouteListFiltered(family int, filter *netlink.Route, mask uint64) ([]netlink.Route, error)
-	GetGvnicNames() ([]string, error)
+	GetStandardNICs() ([]string, error)
 }
 
 type nodeClient interface {
 	GetNodeWithRetry(ctx context.Context, nodeName string) (*v1.Node, error)
 }
 
+// MetadataClient abstracts the metadata service.
+type MetadataClient interface {
+	GetNetworkInterfaces() ([]metadata.NetworkInterface, error)
+}
+
 // RouteManager provides methods for network configuration using the netlinker interface.
 type RouteManager struct {
 	nl netlinker
 	nc nodeClient
+	mc MetadataClient
 }
 
 // realNetlink is the real implementation of netlinker,
@@ -89,7 +97,7 @@ func (r *realNetlink) RouteListFiltered(family int, filter *netlink.Route, mask
 	return netlink.RouteListFiltered(family, filter, mask)
 }
 
-func (r *realNetlink) GetGvnicNames() ([]string, error) {
+func (r *realNetlink) GetStandardNICs() ([]string, error) {
 	links, err := netlink.LinkList()
 	if err != nil {
 		return nil, fmt.Errorf("failed to list network interfaces: %w", err)
@@ -136,13 +144,69 @@ func NewK8sClient() nodeClient {
 	return &k8sNodeClient{}
 }
 
-func Manager(nl netlinker, nc nodeClient) *RouteManager {
-	return &RouteManager{nl: nl, nc: nc}
+func Manager(nl netlinker, nc nodeClient, mc MetadataClient) *RouteManager {
+	return &RouteManager{nl: nl, nc: nc, mc: mc}
 }
 
-// GetGvnicNames gets all available nics on the node.
-func (rm *RouteManager) GetGvnicNames() ([]string, error) {
-	return rm.nl.GetGvnicNames()
+// GetStandardNICs gets all standard NICs on the node which are within the same VPC network
+// as the GCE instance primary network.
+func (rm *RouteManager) GetStandardNICs() ([]string, error) {
+	nics, err := rm.nl.GetStandardNICs()
+	if err != nil {
+		return nil, err
+	}
+
+	// If we have 0 or 1 NIC, no need to validate VPC as single NIC is always valid (primary).
+	// Actually, if 1 NIC, it must be primary.
+	if len(nics) <= 1 {
+		return nics, nil
+	}
+
+	if rm.mc == nil {
+		klog.Warning("Metadata client is nil, skipping VPC validation")
+		return nics, nil
+	}
+
+	// Get network interfaces from metadata (cached).
+	metaNics, err := rm.mc.GetNetworkInterfaces()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get network interfaces from metadata: %w", err)
+	}
+
+	if len(metaNics) == 0 {
+		klog.Warning("No network interfaces found in metadata, skipping VPC validation")
+		return nics, nil
+	}
+
+	// The first network interface in the metadata list corresponds to the primary NIC (index 0).
+	primaryNetwork := metaNics[0].Network
+
+	// Iterate over all discovered NICs and validate them against the metadata.
+	// We match NICs by MAC address and ensure they belong to the same VPC network
+	// as the primary interface.
+	validNics := []string{}
+	for _, nic := range nics {
+		link, err := rm.nl.LinkByName(nic)
+		if err != nil {
+			klog.Warningf("Failed to get link for %s: %v", nic, err)
+			continue
+		}
+		mac := link.Attrs().HardwareAddr.String()
+
+		for _, metaNic := range metaNics {
+			if strings.EqualFold(metaNic.Mac, mac) {
+				if metaNic.Network == primaryNetwork {
+					validNics = append(validNics, nic)
+				} else {
+					klog.Infof("Skipping nic %s because it is in a different VPC network: %s (primary: %s)", nic, metaNic.Network, primaryNetwork)
+				}
+				break
+			}
+		}
+	}
+
+	klog.Infof("Filtered NICs: %v (original: %v)", validNics, nics)
+	return validNics, nil
 }
 
 func (rm *RouteManager) configureRoutesForNic(nicName string, nicIPAddr net.IP, instanceIPAddr string, tableID int) error {