feat(gc): remove old completed virtual machine pods with a garbage collector

images/virtualization-artifact/cmd/virtualization-controller/main.go

@@ -39,6 +39,7 @@ import (
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"github.com/deckhouse/deckhouse/pkg/log"
+
 	appconfig "github.com/deckhouse/virtualization-controller/pkg/config"
 	"github.com/deckhouse/virtualization-controller/pkg/controller/cvi"
 	"github.com/deckhouse/virtualization-controller/pkg/controller/evacuation"
@@ -326,7 +327,15 @@ func main() {
 		log.Error(err.Error())
 		os.Exit(1)
 	}
-	if err = vm.SetupGC(mgr, vmLogger, gcSettings.VMIMigration); err != nil {
+
+	gcMIGLogger := logger.NewControllerLogger(vm.GCVMMigrationControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList)
+	if err = vm.SetupGCMigrations(mgr, gcMIGLogger, gcSettings.VMIMigration); err != nil {
+		log.Error(err.Error())
+		os.Exit(1)
+	}
+
+	gcPODGLogger := logger.NewControllerLogger(vm.GCCompletedPodControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList)
+	if err = vm.SetupGCCompletedPods(mgr, gcPODGLogger, gcSettings.CompletedPod); err != nil {
 		log.Error(err.Error())
 		os.Exit(1)
 	}

images/virtualization-artifact/pkg/config/load_gc_settings.go

@@ -29,11 +29,13 @@ const (
 	GcVmopScheduleVar         = "GC_VMOP_SCHEDULE"
 	GcVMIMigrationTTLVar      = "GC_VMI_MIGRATION_TTL"
 	GcVMIMigrationScheduleVar = "GC_VMI_MIGRATION_SCHEDULE"
+	GcCompletedPodScheduleVar = "GC_COMPLETED_POD_SCHEDULE"
 )
 
 type GCSettings struct {
 	VMOP         BaseGcSettings
 	VMIMigration BaseGcSettings
+	CompletedPod BaseGcSettings
 }
 
 type BaseGcSettings struct {
@@ -55,6 +57,12 @@ func LoadGcSettings() (GCSettings, error) {
 	}
 	gcSettings.VMIMigration = base
 
+	base, err = GetBaseGCSettingsFromEnv(GcCompletedPodScheduleVar, "")
+	if err != nil {
+		return gcSettings, err
+	}
+	gcSettings.CompletedPod = base
+
 	return gcSettings, nil
 }
 

images/virtualization-artifact/pkg/controller/gc/filter.go

@@ -56,17 +56,24 @@ func DefaultFilter(objs []client.Object, isCandidate IsCandidate, ttl time.Durat
 	}
 
 	result := expired
+	result = append(result, KeepLastRemoveOld(nonExpired, indexFunc, maxCount, now)...)
+
+	return result
+}
+
+func KeepLastRemoveOld(objs []client.Object, indexFunc IndexFunc, maxCount int, now time.Time) []client.Object {
 	if maxCount <= 0 {
-		return result
+		return nil
 	}
 
-	slices.SortFunc(nonExpired, func(a, b client.Object) int {
+	slices.SortFunc(objs, func(a, b client.Object) int {
 		return cmp.Compare(getAge(a, now), getAge(b, now))
 	})
 
-	// Keep maxCount first items (most recently created) for each common index. Index example: virtual machine name.
+	var result []client.Object
+
 	indexed := make(map[string]int)
-	for _, obj := range nonExpired {
+	for _, obj := range objs {
 		index := indexFunc(obj)
 		count := indexed[index]
 		if count >= maxCount {

images/virtualization-artifact/pkg/controller/vm/gc.go

@@ -20,18 +20,25 @@ import (
 	"context"
 	"time"
 
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	virtv1 "kubevirt.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	"github.com/deckhouse/deckhouse/pkg/log"
+
 	"github.com/deckhouse/virtualization-controller/pkg/config"
 	"github.com/deckhouse/virtualization-controller/pkg/controller/gc"
 )
 
-const gcVMMigrationControllerName = "vmi-migration-gc-controller"
+const (
+	GCVMMigrationControllerName  = "vmi-migration-gc-controller"
+	GCCompletedPodControllerName = "completed-pod-gc-controller"
+)
 
-func SetupGC(
+func SetupGCMigrations(
 	mgr manager.Manager,
 	log *log.Logger,
 	gcSettings config.BaseGcSettings,
@@ -42,15 +49,32 @@ func SetupGC(
 	if err != nil {
 		return err
 	}
-
-	return gc.SetupGcController(gcVMMigrationControllerName,
+	return gc.SetupGcController(GCVMMigrationControllerName,
 		mgr,
 		log,
 		source,
 		vmimGCMgr,
 	)
 }
 
+func SetupGCCompletedPods(
+	mgr manager.Manager,
+	log *log.Logger,
+	gcSettings config.BaseGcSettings,
+) error {
+	podGCMgr := newCompletedPodGCManager(mgr.GetClient(), 10)
+	source, err := gc.NewCronSource(gcSettings.Schedule, podGCMgr, log.With("resource", "pod"))
+	if err != nil {
+		return err
+	}
+	return gc.SetupGcController(GCCompletedPodControllerName,
+		mgr,
+		log,
+		source,
+		podGCMgr,
+	)
+}
+
 func newVMIMGCManager(client client.Client, ttl time.Duration, max int) *vmimGCManager {
 	if ttl == 0 {
 		ttl = 24 * time.Hour
@@ -105,9 +129,70 @@ func (m *vmimGCManager) getIndex(obj client.Object) string {
 	if !ok {
 		return ""
 	}
-	return vmim.Spec.VMIName
+	return types.NamespacedName{Namespace: vmim.GetNamespace(), Name: vmim.Spec.VMIName}.String()
 }
 
 func vmiMigrationIsFinal(migration *virtv1.VirtualMachineInstanceMigration) bool {
 	return migration.Status.Phase == virtv1.MigrationFailed || migration.Status.Phase == virtv1.MigrationSucceeded
 }
+
+type completedPodGCmanager struct {
+	client client.Client
+	max    int
+}
+
+func newCompletedPodGCManager(client client.Client, max int) *completedPodGCmanager {
+	if max == 0 {
+		max = 10
+	}
+	return &completedPodGCmanager{
+		client: client,
+		max:    max,
+	}
+}
+
+func (m *completedPodGCmanager) New() client.Object {
+	return &corev1.Pod{}
+}
+
+func (m *completedPodGCmanager) ShouldBeDeleted(obj client.Object) bool {
+	pod, ok := obj.(*corev1.Pod)
+	if !ok {
+		return false
+	}
+
+	return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed
+}
+
+func (m *completedPodGCmanager) ListForDelete(ctx context.Context, now time.Time) ([]client.Object, error) {
+	podList := &corev1.PodList{}
+	err := m.client.List(ctx, podList, client.MatchingLabels{
+		"kubevirt.internal.virtualization.deckhouse.io": "virt-launcher",
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var objs []client.Object
+	for _, pod := range podList.Items {
+		if m.ShouldBeDeleted(&pod) {
+			objs = append(objs, &pod)
+		}
+	}
+
+	result := gc.KeepLastRemoveOld(objs, m.getIndex, m.max, now)
+
+	return result, nil
+}
+
+func (m *completedPodGCmanager) getIndex(obj client.Object) string {
+	pod, ok := obj.(*corev1.Pod)
+	if !ok {
+		return ""
+	}
+	owner := metav1.GetControllerOf(pod)
+	if owner != nil {
+		return string(owner.UID)
+	}
+	return ""
+}

images/virtualization-artifact/pkg/controller/vmop/gc.go

@@ -20,10 +20,12 @@ import (
 	"context"
 	"time"
 
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	"github.com/deckhouse/deckhouse/pkg/log"
+
 	commonvmop "github.com/deckhouse/virtualization-controller/pkg/common/vmop"
 	"github.com/deckhouse/virtualization-controller/pkg/config"
 	"github.com/deckhouse/virtualization-controller/pkg/controller/gc"
@@ -106,5 +108,5 @@ func (m *vmopGCManager) getIndex(obj client.Object) string {
 	if !ok {
 		return ""
 	}
-	return vmop.Spec.VirtualMachine
+	return types.NamespacedName{Namespace: vmop.GetNamespace(), Name: vmop.Spec.VirtualMachine}.String()
 }

templates/virtualization-controller/_helpers.tpl

@@ -78,6 +78,8 @@ true
   value: "24h"
 - name: GC_VMI_MIGRATION_SCHEDULE
   value: "0 0 * * *"
+- name: GC_COMPLETED_POD_SCHEDULE
+  value: "0 0 * * *"
 {{- if (hasKey .Values.virtualization.internal.moduleConfig "liveMigration") }}
 - name: LIVE_MIGRATION_BANDWIDTH_PER_NODE
   value: {{ .Values.virtualization.internal.moduleConfig.liveMigration.bandwidthPerNode | quote }}