Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 79 additions & 6 deletions pkg/controller/vsphere/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,15 @@ const (
// Not all controllers support up to 30, but the maximum is 30.
// xref: https://docs.vmware.com/en/VMware-vSphere/8.0/vsphere-vm-administration/GUID-5872D173-A076-42FE-8D0B-9DB0EB0E7362.html#:~:text=If%20you%20add%20a%20hard,values%20from%200%20to%2014.
maxUnitNumber = 30
// VSphereCSIDriverName is the CSI driver name for vSphere volumes.
VSphereCSIDriverName = "csi.vsphere.vmware.com"
// VSphereInTreePluginName is the in-tree plugin name for vSphere volumes.
VSphereInTreePluginName = "kubernetes.io/vsphere-volume"
// csiPluginPrefix is the prefix for CSI volume names in node.Status.VolumesAttached.
// Format: kubernetes.io/csi/<driverName>^<volumeHandle>
csiPluginPrefix = "kubernetes.io/csi/"
// csiVolNameSep separates the driver name from the volume handle in CSI volume names.
csiVolNameSep = "^"
)

// These are the guestinfo variables used by Ignition.
Expand Down Expand Up @@ -534,11 +543,11 @@ func (r *Reconciler) delete() error {
return fmt.Errorf("destroying vm in progress, requeuing")
}

// nodeHasVolumesAttached returns true if node status still have volumes attached
// pod deletion and volume detach happen asynchronously, so pod could be deleted before volume detached from the node
// this could cause issue for some storage provisioner, for example, vsphere-volume this is problematic
// because if the node is deleted before detach success, then the underline VMDK will be deleted together with the Machine
// so after node draining we need to check if all volumes are detached before deleting the node.
// nodeHasVolumesAttached returns true if node status still has vSphere-backed volumes attached.
// Pod deletion and volume detach happen asynchronously, so pod could be deleted before volume detached from the node.
// This is problematic for vSphere volumes because if the node is deleted before detach succeeds,
// the underlying VMDK will be deleted together with the Machine.
// Non-vSphere volumes (NFS, iSCSI, etc.) do not have this risk since vSphere Destroy_Task does not affect them.
func (r *Reconciler) nodeHasVolumesAttached(ctx context.Context, nodeName string, machineName string) (bool, error) {
node := &corev1.Node{}
if err := r.apiReader.Get(ctx, apimachinerytypes.NamespacedName{Name: nodeName}, node); err != nil {
Expand All @@ -549,7 +558,71 @@ func (r *Reconciler) nodeHasVolumesAttached(ctx context.Context, nodeName string
return true, err
}

return len(node.Status.VolumesAttached) != 0, nil
if len(node.Status.VolumesAttached) == 0 {
return false, nil
}

klog.V(3).Infof("Machine %s: checking %d attached volumes on node %s for vSphere-backed volumes", machineName, len(node.Status.VolumesAttached), nodeName)

var vsphereVolumes []string
var nonVSphereVolumes []string
var unknownVolumes []string

for _, vol := range node.Status.VolumesAttached {
volName := string(vol.Name)

switch classifyAttachedVolume(volName) {
case volumeClassVSphere:
vsphereVolumes = append(vsphereVolumes, volName)
case volumeClassNonVSphere:
nonVSphereVolumes = append(nonVSphereVolumes, volName)
case volumeClassUnknown:
unknownVolumes = append(unknownVolumes, volName)
}
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

if len(vsphereVolumes) > 0 {
klog.Warningf("Machine %s: vSphere-backed volumes still attached on node %s: %v", machineName, nodeName, vsphereVolumes)
}
if len(nonVSphereVolumes) > 0 {
klog.V(3).Infof("Machine %s: non-vSphere volumes attached (safe to ignore): %v", machineName, nonVSphereVolumes)
}
if len(unknownVolumes) > 0 {
klog.Warningf("Machine %s on node %s: volumes with unrecognized name format, conservatively blocking deletion: %v", machineName, nodeName, unknownVolumes)
}

return len(vsphereVolumes) > 0 || len(unknownVolumes) > 0, nil
}

type volumeClass int

const (
volumeClassVSphere volumeClass = iota
volumeClassNonVSphere
volumeClassUnknown
)

// classifyAttachedVolume determines whether an attached volume is vSphere-backed
// by parsing the UniqueVolumeName format from node.Status.VolumesAttached.
// CSI volumes use the format: kubernetes.io/csi/<driverName>^<volumeHandle>
// In-tree vSphere volumes use the prefix: kubernetes.io/vsphere-volume/
func classifyAttachedVolume(volName string) volumeClass {
if strings.HasPrefix(volName, VSphereInTreePluginName+"/") {
return volumeClassVSphere
}

if strings.HasPrefix(volName, csiPluginPrefix) {
remainder := strings.TrimPrefix(volName, csiPluginPrefix)
if sepIdx := strings.Index(remainder, csiVolNameSep); sepIdx > 0 {
driver := remainder[:sepIdx]
if driver == VSphereCSIDriverName {
return volumeClassVSphere
}
return volumeClassNonVSphere
}
}

return volumeClassUnknown
}

// reconcileMachineWithCloudState reconcile machineSpec and status with the latest cloud state
Expand Down
Loading