feat(vmop): surface "quota exceeded" error during migration (#1310)

Signed-off-by: Valeriy Khorunzhin <[email protected]>

Author: eofff

api/core/v1alpha2/events.go

@@ -58,6 +58,9 @@ const (
 	// ReasonErrVMOPFailed is event reason that operation is failed
 	ReasonErrVMOPFailed = "VirtualMachineOperationFailed"
 
+	// ReasonErrVMOPPending is event reason that operation is pending
+	ReasonErrVMOPPending = "VirtualMachineOperationPending"
+
 	// ReasonVMOPSucceeded is event reason that the operation is successfully completed
 	ReasonVMOPSucceeded = "VirtualMachineOperationSucceeded"
 

api/core/v1alpha2/vmcondition/condition.go

@@ -92,6 +92,7 @@ const (
 	ReasonVmIsRunning                    Reason = "VirtualMachineRunning"
 	ReasonInternalVirtualMachineError    Reason = "InternalVirtualMachineError"
 	ReasonPodNotStarted                  Reason = "PodNotStarted"
+	ReasonMigrationIsPending             Reason = "MigrationIsPending"
 
 	// 	ReasonFilesystemFrozen indicates that virtual machine's filesystem has been successfully frozen.
 	ReasonFilesystemFrozen Reason = "Frozen"

api/core/v1alpha2/vmopcondition/condition.go

@@ -80,6 +80,9 @@ const (
 	// ReasonOtherMigrationInProgress is a ReasonCompleted indicating that there are other migrations in progress.
 	ReasonOtherMigrationInProgress ReasonCompleted = "OtherMigrationInProgress"
 
+	// ReasonQuotaExceeded is a completed reason that indicates the project's quota has been exceeded and the migration has been paused.
+	ReasonQuotaExceeded ReasonCompleted = "QuotaExceeded"
+
 	// ReasonOperationFailed is a ReasonCompleted indicating that operation has failed.
 	ReasonOperationFailed ReasonCompleted = "OperationFailed"
 

images/virtualization-artifact/pkg/controller/conditions/getter.go

@@ -66,7 +66,18 @@ func GetKVVMICondition(condType virtv1.VirtualMachineInstanceConditionType, cond
 	return virtv1.VirtualMachineInstanceCondition{}, false
 }
 
+func GetKVVMIMCondition(condType virtv1.VirtualMachineInstanceMigrationConditionType, conditions []virtv1.VirtualMachineInstanceMigrationCondition) (virtv1.VirtualMachineInstanceMigrationCondition, bool) {
+	for _, condition := range conditions {
+		if condition.Type == condType {
+			return condition, true
+		}
+	}
+
+	return virtv1.VirtualMachineInstanceMigrationCondition{}, false
+}
+
 const (
-	VirtualMachineInstanceNodePlacementNotMatched virtv1.VirtualMachineInstanceConditionType = "NodePlacementNotMatched"
-	VirtualMachineSynchronized                    virtv1.VirtualMachineConditionType         = "Synchronized"
+	VirtualMachineInstanceNodePlacementNotMatched virtv1.VirtualMachineInstanceConditionType          = "NodePlacementNotMatched"
+	KubevirtMigrationRejectedByResourceQuotaType  virtv1.VirtualMachineInstanceMigrationConditionType = "migrationRejectedByResourceQuota"
+	VirtualMachineSynchronized                    virtv1.VirtualMachineConditionType                  = "Synchronized"
 )

images/virtualization-artifact/pkg/controller/vm/internal/migrating.go

@@ -18,6 +18,7 @@ package internal
 
 import (
 	"context"
+	"fmt"
 	"log/slog"
 
 	corev1 "k8s.io/api/core/v1"
@@ -136,7 +137,7 @@ func (h *MigratingHandler) syncMigrating(vm *virtv2.VirtualMachine, kvvmi *virtv
 	{
 		var inProgressVmops []*virtv2.VirtualMachineOperation
 		for _, op := range vmops {
-			if commonvmop.IsMigration(op) && op.Status.Phase == virtv2.VMOPPhaseInProgress {
+			if commonvmop.IsMigration(op) && isOperationInProgress(op) {
 				inProgressVmops = append(inProgressVmops, op)
 			}
 		}
@@ -167,6 +168,9 @@ func (h *MigratingHandler) syncMigrating(vm *virtv2.VirtualMachine, kvvmi *virtv
 			cb.Message("Migration is awaiting execution.")
 		case vmopcondition.ReasonMigrationRunning.String():
 			cb.Status(metav1.ConditionTrue).Reason(vmcondition.ReasonVmIsMigrating)
+		case vmopcondition.ReasonQuotaExceeded.String():
+			cb.Reason(vmcondition.ReasonMigrationIsPending)
+			cb.Message(fmt.Sprintf("Migration is pending: %s", completed.Message))
 		}
 		conditions.SetCondition(cb, &vm.Status.Conditions)
 
@@ -203,3 +207,9 @@ func liveMigrationInProgress(migrationState *virtv2.VirtualMachineMigrationState
 func liveMigrationFailed(migrationState *virtv2.VirtualMachineMigrationState) bool {
 	return migrationState != nil && migrationState.EndTimestamp != nil && migrationState.Result == virtv2.MigrationResultFailed
 }
+
+func isOperationInProgress(vmop *virtv2.VirtualMachineOperation) bool {
+	sent, _ := conditions.GetCondition(vmopcondition.TypeSignalSent, vmop.Status.Conditions)
+	completed, _ := conditions.GetCondition(vmopcondition.TypeCompleted, vmop.Status.Conditions)
+	return sent.Status == metav1.ConditionTrue && completed.Status != metav1.ConditionTrue
+}

images/virtualization-artifact/pkg/controller/vm/internal/migrating_test.go

@@ -66,7 +66,7 @@ var _ = Describe("MigratingHandler", func() {
 		return kvvmi
 	}
 
-	newVMOP := func(phase virtv2.VMOPPhase, reason string) *virtv2.VirtualMachineOperation {
+	newVMOP := func(phase virtv2.VMOPPhase, reason string, isSignalSent bool) *virtv2.VirtualMachineOperation {
 		vmop := vmopbuilder.New(
 			vmopbuilder.WithGenerateName("test-vmop-"),
 			vmopbuilder.WithNamespace(namespace),
@@ -81,6 +81,12 @@ var _ = Describe("MigratingHandler", func() {
 				Reason: reason,
 			},
 		}
+		if isSignalSent {
+			vmop.Status.Conditions = append(vmop.Status.Conditions, metav1.Condition{
+				Type:   vmopcondition.TypeSignalSent.String(),
+				Status: metav1.ConditionTrue,
+			})
+		}
 		return vmop
 	}
 
@@ -160,7 +166,7 @@ var _ = Describe("MigratingHandler", func() {
 		It("Should set condition when vmop is in progress with pending reason", func() {
 			vm := newVM()
 			kvvmi := newKVVMI(nil)
-			vmop := newVMOP(virtv2.VMOPPhaseInProgress, vmopcondition.ReasonMigrationPending.String())
+			vmop := newVMOP(virtv2.VMOPPhaseInProgress, vmopcondition.ReasonMigrationPending.String(), true)
 			fakeClient, resource, vmState = setupEnvironment(vm, kvvmi, vmop)
 
 			reconcile()
@@ -179,7 +185,7 @@ var _ = Describe("MigratingHandler", func() {
 		It("Should set condition when vmop is in progress with target ready reason", func() {
 			vm := newVM()
 			kvvmi := newKVVMI(nil)
-			vmop := newVMOP(virtv2.VMOPPhaseInProgress, vmopcondition.ReasonMigrationTargetReady.String())
+			vmop := newVMOP(virtv2.VMOPPhaseInProgress, vmopcondition.ReasonMigrationTargetReady.String(), true)
 			fakeClient, resource, vmState = setupEnvironment(vm, kvvmi, vmop)
 
 			reconcile()
@@ -198,7 +204,7 @@ var _ = Describe("MigratingHandler", func() {
 		It("Should set condition when vmop is in progress with running reason", func() {
 			vm := newVM()
 			kvvmi := newKVVMI(nil)
-			vmop := newVMOP(virtv2.VMOPPhaseInProgress, vmopcondition.ReasonMigrationRunning.String())
+			vmop := newVMOP(virtv2.VMOPPhaseInProgress, vmopcondition.ReasonMigrationRunning.String(), true)
 			fakeClient, resource, vmState = setupEnvironment(vm, kvvmi, vmop)
 
 			reconcile()

images/virtualization-artifact/pkg/controller/vm/internal/watcher/vmop_watcher.go

@@ -61,13 +61,6 @@ func (w *VMOPWatcher) Watch(mgr manager.Manager, ctr controller.Controller) erro
 					return commonvmop.IsMigration(e.Object)
 				},
 				UpdateFunc: func(e event.TypedUpdateEvent[*virtv2.VirtualMachineOperation]) bool {
-					if commonvmop.IsMigration(e.ObjectNew) {
-						return false
-					}
-					if e.ObjectNew.Status.Phase != virtv2.VMOPPhaseInProgress {
-						return false
-					}
-
 					oldCompleted, _ := conditions.GetCondition(vmopcondition.TypeCompleted, e.ObjectOld.Status.Conditions)
 					newCompleted, _ := conditions.GetCondition(vmopcondition.TypeCompleted, e.ObjectNew.Status.Conditions)
 

images/virtualization-artifact/pkg/controller/vmop/internal/lifecycle.go

@@ -106,6 +106,22 @@ func (h LifecycleHandler) Handle(ctx context.Context, vmop *virtv2.VirtualMachin
 		return reconcile.Result{}, nil
 	}
 
+	// Pending if quota exceeded.
+	isQuotaExceededDuringMigration, err := service.IsKubeVirtMigrationRejectedDueToQuota(ctx, h.client, vmop)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+	if isQuotaExceededDuringMigration {
+		h.recorder.Event(vmop, corev1.EventTypeWarning, virtv2.ReasonErrVMOPPending, "Project quota exceeded")
+		conditions.SetCondition(
+			completedCond.
+				Reason(vmopcondition.ReasonQuotaExceeded).
+				Status(metav1.ConditionFalse).
+				Message("Project quota exceeded"),
+			&vmop.Status.Conditions)
+		return reconcile.Result{}, nil
+	}
+
 	// Get VM for Pending and InProgress checks.
 	vm, err := object.FetchObject(ctx, types.NamespacedName{Name: vmop.Spec.VirtualMachine, Namespace: vmop.Namespace}, h.client, &virtv2.VirtualMachine{})
 	if err != nil {

images/virtualization-artifact/pkg/controller/vmop/internal/service/migrate.go

@@ -28,6 +28,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/deckhouse/virtualization-controller/pkg/common/object"
+	commonvmop "github.com/deckhouse/virtualization-controller/pkg/common/vmop"
 	"github.com/deckhouse/virtualization-controller/pkg/controller/conditions"
 	virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
 	"github.com/deckhouse/virtualization/api/core/v1alpha2/vmopcondition"
@@ -177,3 +178,29 @@ var mapMigrationPhaseToReason = map[virtv1.VirtualMachineInstanceMigrationPhase]
 	virtv1.MigrationSucceeded:   vmopcondition.ReasonOperationCompleted,
 	virtv1.MigrationFailed:      vmopcondition.ReasonOperationFailed,
 }
+
+func IsKubeVirtMigrationRejectedDueToQuota(ctx context.Context, client client.Client, vmop *virtv2.VirtualMachineOperation) (bool, error) {
+	if !commonvmop.IsMigration(vmop) {
+		return false, nil
+	}
+
+	kubevirtMigrationName := migrationName(vmop)
+	kubevirtMigration, err := object.FetchObject(ctx, types.NamespacedName{
+		Namespace: vmop.GetNamespace(),
+		Name:      kubevirtMigrationName,
+	}, client, &virtv1.VirtualMachineInstanceMigration{})
+	if err != nil {
+		return false, err
+	}
+
+	if kubevirtMigration == nil {
+		return false, nil
+	}
+
+	_, ok := conditions.GetKVVMIMCondition(conditions.KubevirtMigrationRejectedByResourceQuotaType, kubevirtMigration.Status.Conditions)
+	if ok {
+		return true, nil
+	}
+
+	return false, nil
+}

images/virtualization-artifact/pkg/controller/vmop/internal/watcher/migration.go

@@ -21,10 +21,8 @@ import (
 
 	virtv1 "kubevirt.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
-	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	"github.com/deckhouse/virtualization/api/core/v1alpha2"
@@ -45,11 +43,6 @@ func (w MigrationWatcher) Watch(mgr manager.Manager, ctr controller.Controller)
 				&v1alpha2.VirtualMachineOperation{},
 				handler.OnlyControllerOwner(),
 			),
-			predicate.TypedFuncs[*virtv1.VirtualMachineInstanceMigration]{
-				UpdateFunc: func(e event.TypedUpdateEvent[*virtv1.VirtualMachineInstanceMigration]) bool {
-					return e.ObjectOld.Status.Phase != e.ObjectNew.Status.Phase
-				},
-			},
 		),
 	); err != nil {
 		return fmt.Errorf("error setting watch on VirtualMachineInstanceMigration: %w", err)