Delete ONTAP FlexVols by UUID to avoid REST name-index orphans

ONTAP's REST name index is updated asynchronously, so a just-created
FlexVol can be reported as already gone by a name-based lookup. The
destroy path's VolumeExists pre-check then skipped the delete and left
the volume orphaned.

Author: adkerr

core/concurrent_core.go

@@ -3132,6 +3132,8 @@ func (o *ConcurrentTridentOrchestrator) cloneVolume(
 	// Clear these values as they were copied from the source volume Config
 	cloneConfig.SubordinateVolumes = make(map[string]interface{})
 	cloneConfig.ShareSourceVolume = ""
+	// The clone gets its own backing volume, whose ID the driver records when it creates it.
+	cloneConfig.BackendVolumeID = ""
 	cloneConfig.LUKSPassphraseNames = sourceVolume.Config.LUKSPassphraseNames
 	// Override this value only if SplitOnClone has been defined in clone volume's config
 	if volConfig.SplitOnClone != "" {

core/orchestrator_core.go

@@ -2652,6 +2652,8 @@ func (o *TridentOrchestrator) cloneVolumeInitial(
 	// Clear these values as they were copied from the source volume Config
 	cloneConfig.SubordinateVolumes = make(map[string]interface{})
 	cloneConfig.ShareSourceVolume = ""
+	// The clone gets its own backing volume, whose ID the driver records when it creates it.
+	cloneConfig.BackendVolumeID = ""
 	cloneConfig.LUKSPassphraseNames = sourceVolume.Config.LUKSPassphraseNames
 	// Override this value only if SplitOnClone has been defined in clone volume's config
 	if volumeConfig.SplitOnClone != "" {

mocks/mock_storage_drivers/mock_ontap/mock_api.go

@@ -71,7 +71,9 @@ type VolumeConfig struct {
 	// PeerVolumeHandle is the internal volume handle for the source volume if this volume is a mirror destination
 	PeerVolumeHandle string `json:"requiredPeerVolumeHandle,omitempty"`
 	// InternalID is an optional, backend-specific identifier to help find an object
-	InternalID         string                 `json:"internalID,omitempty"`
+	InternalID string `json:"internalID,omitempty"`
+	// BackendVolumeID is a stable, backend-assigned identifier for the volume's backing object.
+	BackendVolumeID    string                 `json:"backendVolumeID,omitempty"`
 	ShareSourceVolume  string                 `json:"shareSourceVolume"`
 	SubordinateVolumes map[string]interface{} `json:"-"`
 	Namespace          string                 `json:"namespace"`

mocks/mock_storage_drivers/mock_ontap/mock_ontap_rest_interface.go

@@ -212,15 +212,23 @@ type OntapAPI interface {
 	VolumeCloneCreate(ctx context.Context, cloneName, sourceName, snapshot string, async bool) error
 	VolumeCloneSplitStart(ctx context.Context, cloneName string) error
 
-	VolumeCreate(ctx context.Context, volume Volume) error
-	VolumeCreateBalanced(ctx context.Context, volume Volume) error
+	// VolumeCreate creates a volume and returns the UUID the backend assigned to it, which is empty
+	// when the backend does not report one (for example on the ZAPI path).
+	VolumeCreate(ctx context.Context, volume Volume) (string, error)
+	VolumeCreateBalanced(ctx context.Context, volume Volume) (string, error)
 	VolumeModify(ctx context.Context, volume Volume) error
 	// VolumeDestroy keeps both flags in the shared abstraction because ZAPI and REST expose
 	// delete semantics differently. ZAPI consumes force directly and may need a follow-up
 	// recovery-queue purge when skipRecoveryQueue is true. REST only exposes one delete flag at
 	// the transport layer, so the REST adapter maps skipRecoveryQueue onto that flag while keeping
 	// this caller-facing signature stable.
 	VolumeDestroy(ctx context.Context, volumeName string, force, skipRecoveryQueue bool) error
+	// VolumeDestroyByUUID deletes the volume identified by volumeUUID. Deleting by UUID avoids the
+	// name index, which ONTAP updates asynchronously and which can therefore report a newly created
+	// volume as missing. It returns a NotFoundError once the UUID no longer resolves, and an
+	// UnsupportedError when the caller must fall back to deleting by name (no UUID provided, or
+	// the ZAPI path which cannot delete by UUID).
+	VolumeDestroyByUUID(ctx context.Context, volumeUUID, volumeName string, force, skipRecoveryQueue bool) error
 	VolumeMove(
 		ctx context.Context, volumeName, destinationAggregateName string, dryRun bool,
 	) (string, error)

storage/volume.go

@@ -174,7 +174,7 @@ func (d OntapAPIREST) ValidateAPIVersion(ctx context.Context) error {
 	return nil
 }
 
-func (d OntapAPIREST) VolumeCreate(ctx context.Context, volume Volume) error {
+func (d OntapAPIREST) VolumeCreate(ctx context.Context, volume Volume) (string, error) {
 	fields := LogFields{
 		"Method": "VolumeCreate",
 		"Type":   "OntapAPIREST",
@@ -190,17 +190,17 @@ func (d OntapAPIREST) VolumeCreate(ctx context.Context, volume Volume) error {
 		aggregateName = volume.Aggregates[0]
 	}
 
-	creationErr := d.api.VolumeCreate(ctx, volume.Name, aggregateName, volume.Size, volume.SpaceReserve,
+	volumeUUID, creationErr := d.api.VolumeCreate(ctx, volume.Name, aggregateName, volume.Size, volume.SpaceReserve,
 		volume.SnapshotPolicy, volume.UnixPermissions, volume.ExportPolicy, volume.SecurityStyle,
 		volume.TieringPolicy, volume.Comment, volume.Qos, volume.Encrypt, volume.SnapshotReserve, volume.DPVolume)
 	if creationErr != nil {
-		return fmt.Errorf("error creating volume: %v", creationErr)
+		return "", fmt.Errorf("error creating volume: %v", creationErr)
 	}
 
-	return nil
+	return volumeUUID, nil
 }
 
-func (d OntapAPIREST) VolumeCreateBalanced(ctx context.Context, volume Volume) error {
+func (d OntapAPIREST) VolumeCreateBalanced(ctx context.Context, volume Volume) (string, error) {
 	fields := LogFields{
 		"Method": "VolumeCreateBalanced",
 		"Type":   "OntapAPIREST",
@@ -213,17 +213,17 @@ func (d OntapAPIREST) VolumeCreateBalanced(ctx context.Context, volume Volume) e
 
 	// Double-check this API is supported
 	if !d.SupportsFeature(ctx, BalancedPlacement) {
-		return errors.UnsupportedError("ONTAP version does not support balanced placement")
+		return "", errors.UnsupportedError("ONTAP version does not support balanced placement")
 	}
 
-	creationErr := d.api.VolumeCreateBalanced(ctx, volume.Name, volume.Size, volume.SpaceReserve,
+	volumeUUID, creationErr := d.api.VolumeCreateBalanced(ctx, volume.Name, volume.Size, volume.SpaceReserve,
 		volume.SnapshotPolicy, volume.UnixPermissions, volume.ExportPolicy, volume.SecurityStyle,
 		volume.TieringPolicy, volume.Comment, volume.Qos, volume.Encrypt, volume.SnapshotReserve, volume.DPVolume)
 	if creationErr != nil {
-		return fmt.Errorf("error creating volume: %v", creationErr)
+		return "", fmt.Errorf("error creating volume: %v", creationErr)
 	}
 
-	return nil
+	return volumeUUID, nil
 }
 
 func (d OntapAPIREST) VolumeModify(ctx context.Context, volume Volume) error {
@@ -233,14 +233,61 @@ func (d OntapAPIREST) VolumeModify(ctx context.Context, volume Volume) error {
 func (d OntapAPIREST) VolumeDestroy(ctx context.Context, name string, force, skipRecoveryQueue bool) error {
 	destroyVolume := func() error {
 		deletionErr := d.api.VolumeDestroy(ctx, name, skipRecoveryQueue)
-		if deletionErr != nil && !IsVolumeBusyRESTError(deletionErr) {
-			return backoff.Permanent(fmt.Errorf("error destroying volume %v: %v", name, deletionErr))
-		}
-		if deletionErr != nil {
-			return fmt.Errorf("error destroying volume %v: %v", name, deletionErr)
+		return classifyVolumeDeleteError(name, deletionErr)
+	}
+	return d.retryVolumeDestroy(ctx, name, destroyVolume)
+}
+
+// VolumeDestroyByUUID deletes the volume identified by volumeUUID. Addressing the volume by UUID
+// avoids the lag in ONTAP's name index, which can report a just-created volume as missing. A UUID
+// that no longer resolves yields a NotFoundError, meaning the delete already completed.
+func (d OntapAPIREST) VolumeDestroyByUUID(
+	ctx context.Context, volumeUUID, volumeName string, force, skipRecoveryQueue bool,
+) error {
+	fields := LogFields{
+		"Method": "VolumeDestroyByUUID",
+		"Type":   "OntapAPIREST",
+		"volume": volumeName,
+		"uuid":   volumeUUID,
+	}
+	Logd(ctx, d.driverName,
+		d.api.ClientConfig().DebugTraceFlags["method"]).WithFields(fields).Trace(">>>> VolumeDestroyByUUID")
+	defer Logd(ctx, d.driverName,
+		d.api.ClientConfig().DebugTraceFlags["method"]).WithFields(fields).Trace("<<<< VolumeDestroyByUUID")
+
+	if volumeUUID == "" {
+		return errors.UnsupportedError("no volume UUID provided; delete by name")
+	}
+
+	destroyVolume := func() error {
+		deletionErr := d.api.VolumeDestroyByUUID(ctx, volumeUUID, skipRecoveryQueue)
+		if errors.IsNotFoundError(deletionErr) {
+			return backoff.Permanent(errors.NotFoundError("volume %s (UUID %s) does not exist", volumeName, volumeUUID))
 		}
+		return classifyVolumeDeleteError(volumeName, deletionErr)
+	}
+	return d.retryVolumeDestroy(ctx, volumeName, destroyVolume)
+}
+
+// classifyVolumeDeleteError wraps a raw delete error so that a busy volume is retried while any
+// other failure aborts the retry loop immediately. A NotFoundError passes through with its type
+// intact, since callers rely on it to tell an already-deleted volume from a real failure.
+func classifyVolumeDeleteError(name string, deletionErr error) error {
+	if deletionErr == nil {
 		return nil
 	}
+	if errors.IsNotFoundError(deletionErr) {
+		return backoff.Permanent(deletionErr)
+	}
+	if !IsVolumeBusyRESTError(deletionErr) {
+		return backoff.Permanent(fmt.Errorf("error destroying volume %v: %v", name, deletionErr))
+	}
+	return fmt.Errorf("error destroying volume %v: %v", name, deletionErr)
+}
+
+// retryVolumeDestroy runs a delete operation with the shared busy-volume backoff, unwrapping the
+// permanent error (which may be a NotFoundError the caller treats as success) when the loop stops.
+func (d OntapAPIREST) retryVolumeDestroy(ctx context.Context, name string, destroyVolume func() error) error {
 	destroyNotify := func(err error, duration time.Duration) {
 		Logc(ctx).WithField("increment", duration).Debug("Volume busy, waiting.")
 	}