From d5c92cad5150158d55e2f11a21d86ad92485046e Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Mon, 8 Sep 2025 09:23:59 +0300 Subject: [PATCH 1/3] delete old completed pods for vm Signed-off-by: Yaroslav Borbat --- .../cmd/virtualization-controller/main.go | 11 ++- .../pkg/config/load_gc_settings.go | 9 ++ .../pkg/controller/vm/gc.go | 98 ++++++++++++++++++- .../pkg/controller/vmop/gc.go | 4 +- .../virtualization-controller/_helpers.tpl | 4 + 5 files changed, 119 insertions(+), 7 deletions(-) diff --git a/images/virtualization-artifact/cmd/virtualization-controller/main.go b/images/virtualization-artifact/cmd/virtualization-controller/main.go index 3609029a02..f80a8e111b 100644 --- a/images/virtualization-artifact/cmd/virtualization-controller/main.go +++ b/images/virtualization-artifact/cmd/virtualization-controller/main.go @@ -39,6 +39,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/deckhouse/deckhouse/pkg/log" + appconfig "github.com/deckhouse/virtualization-controller/pkg/config" "github.com/deckhouse/virtualization-controller/pkg/controller/cvi" "github.com/deckhouse/virtualization-controller/pkg/controller/evacuation" @@ -326,7 +327,15 @@ func main() { log.Error(err.Error()) os.Exit(1) } - if err = vm.SetupGC(mgr, vmLogger, gcSettings.VMIMigration); err != nil { + + gcMIGLogger := logger.NewControllerLogger(vm.GCVMMigrationControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList) + if err = vm.SetupGCMigrations(mgr, gcMIGLogger, gcSettings.VMIMigration); err != nil { + log.Error(err.Error()) + os.Exit(1) + } + + gcPODGLogger := logger.NewControllerLogger(vm.GCCompletedPodControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList) + if err = vm.SetupGCCompletedPods(mgr, gcPODGLogger, gcSettings.CompletedPod); err != nil { log.Error(err.Error()) os.Exit(1) } diff --git a/images/virtualization-artifact/pkg/config/load_gc_settings.go b/images/virtualization-artifact/pkg/config/load_gc_settings.go index 7cac99823f..343b71101d 100644 --- a/images/virtualization-artifact/pkg/config/load_gc_settings.go +++ b/images/virtualization-artifact/pkg/config/load_gc_settings.go @@ -29,11 +29,14 @@ const ( GcVmopScheduleVar = "GC_VMOP_SCHEDULE" GcVMIMigrationTTLVar = "GC_VMI_MIGRATION_TTL" GcVMIMigrationScheduleVar = "GC_VMI_MIGRATION_SCHEDULE" + GcCompletedPodTTLVar = "GC_COMPLETED_POD_TTL" + GcCompletedPodScheduleVar = "GC_COMPLETED_POD_SCHEDULE" ) type GCSettings struct { VMOP BaseGcSettings VMIMigration BaseGcSettings + CompletedPod BaseGcSettings } type BaseGcSettings struct { @@ -55,6 +58,12 @@ func LoadGcSettings() (GCSettings, error) { } gcSettings.VMIMigration = base + base, err = GetBaseGCSettingsFromEnv(GcCompletedPodScheduleVar, GcCompletedPodTTLVar) + if err != nil { + return gcSettings, err + } + gcSettings.CompletedPod = base + return gcSettings, nil } diff --git a/images/virtualization-artifact/pkg/controller/vm/gc.go b/images/virtualization-artifact/pkg/controller/vm/gc.go index 5ccdc8bcdd..7d42a6ae63 100644 --- a/images/virtualization-artifact/pkg/controller/vm/gc.go +++ b/images/virtualization-artifact/pkg/controller/vm/gc.go @@ -20,18 +20,25 @@ import ( "context" "time" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/virtualization-controller/pkg/config" "github.com/deckhouse/virtualization-controller/pkg/controller/gc" ) -const gcVMMigrationControllerName = "vmi-migration-gc-controller" +const ( + GCVMMigrationControllerName = "vmi-migration-gc-controller" + GCCompletedPodControllerName = "completed-pod-gc-controller" +) -func SetupGC( +func SetupGCMigrations( mgr manager.Manager, log *log.Logger, gcSettings config.BaseGcSettings, @@ -42,8 +49,7 @@ func SetupGC( if err != nil { return err } - - return gc.SetupGcController(gcVMMigrationControllerName, + return gc.SetupGcController(GCVMMigrationControllerName, mgr, log, source, @@ -51,6 +57,24 @@ func SetupGC( ) } +func SetupGCCompletedPods( + mgr manager.Manager, + log *log.Logger, + gcSettings config.BaseGcSettings, +) error { + podGCMgr := newCompletedPodGCManager(mgr.GetClient(), gcSettings.TTL.Duration, 10) + source, err := gc.NewCronSource(gcSettings.Schedule, podGCMgr, log.With("resource", "pod")) + if err != nil { + return err + } + return gc.SetupGcController(GCCompletedPodControllerName, + mgr, + log, + source, + podGCMgr, + ) +} + func newVMIMGCManager(client client.Client, ttl time.Duration, max int) *vmimGCManager { if ttl == 0 { ttl = 24 * time.Hour @@ -105,9 +129,73 @@ func (m *vmimGCManager) getIndex(obj client.Object) string { if !ok { return "" } - return vmim.Spec.VMIName + return types.NamespacedName{Namespace: vmim.GetNamespace(), Name: vmim.GetName()}.String() } func vmiMigrationIsFinal(migration *virtv1.VirtualMachineInstanceMigration) bool { return migration.Status.Phase == virtv1.MigrationFailed || migration.Status.Phase == virtv1.MigrationSucceeded } + +type completedPodGCmanager struct { + client client.Client + ttl time.Duration + max int +} + +func newCompletedPodGCManager(client client.Client, ttl time.Duration, max int) *completedPodGCmanager { + if ttl == 0 { + ttl = 24 * time.Hour + } + if max == 0 { + max = 10 + } + return &completedPodGCmanager{ + client: client, + ttl: ttl, + max: max, + } +} + +func (m *completedPodGCmanager) New() client.Object { + return &corev1.Pod{} +} + +func (m *completedPodGCmanager) ShouldBeDeleted(obj client.Object) bool { + pod, ok := obj.(*corev1.Pod) + if !ok { + return false + } + + return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed +} + +func (m *completedPodGCmanager) ListForDelete(ctx context.Context, now time.Time) ([]client.Object, error) { + podList := &corev1.PodList{} + err := m.client.List(ctx, podList, client.MatchingLabels{ + "kubevirt.internal.virtualization.deckhouse.io": "virt-launcher", + }) + if err != nil { + return nil, err + } + + objs := make([]client.Object, 0, len(podList.Items)) + for _, pod := range podList.Items { + objs = append(objs, &pod) + } + + result := gc.DefaultFilter(objs, m.ShouldBeDeleted, m.ttl, m.getIndex, m.max, now) + + return result, nil +} + +func (m *completedPodGCmanager) getIndex(obj client.Object) string { + pod, ok := obj.(*corev1.Pod) + if !ok { + return "" + } + owner := metav1.GetControllerOf(pod) + if owner != nil { + return string(owner.UID) + } + return "" +} diff --git a/images/virtualization-artifact/pkg/controller/vmop/gc.go b/images/virtualization-artifact/pkg/controller/vmop/gc.go index 567d6977f1..b2192887a6 100644 --- a/images/virtualization-artifact/pkg/controller/vmop/gc.go +++ b/images/virtualization-artifact/pkg/controller/vmop/gc.go @@ -20,10 +20,12 @@ import ( "context" "time" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/deckhouse/deckhouse/pkg/log" + commonvmop "github.com/deckhouse/virtualization-controller/pkg/common/vmop" "github.com/deckhouse/virtualization-controller/pkg/config" "github.com/deckhouse/virtualization-controller/pkg/controller/gc" @@ -106,5 +108,5 @@ func (m *vmopGCManager) getIndex(obj client.Object) string { if !ok { return "" } - return vmop.Spec.VirtualMachine + return types.NamespacedName{Namespace: vmop.GetNamespace(), Name: vmop.GetName()}.String() } diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index a8f6abb981..6d8ff22b2f 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -78,6 +78,10 @@ true value: "24h" - name: GC_VMI_MIGRATION_SCHEDULE value: "0 0 * * *" +- name: GC_COMPLETED_POD_TTL + value: "24h" +- name: GC_COMPLETED_POD_SCHEDULE + value: "0 0 * * *" {{- if (hasKey .Values.virtualization.internal.moduleConfig "liveMigration") }} - name: LIVE_MIGRATION_BANDWIDTH_PER_NODE value: {{ .Values.virtualization.internal.moduleConfig.liveMigration.bandwidthPerNode | quote }} From ccb7e18b5bd5b1cc4086af2508a0d12adf0cca1e Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Mon, 8 Sep 2025 09:28:21 +0300 Subject: [PATCH 2/3] fix Signed-off-by: Yaroslav Borbat --- images/virtualization-artifact/pkg/controller/vm/gc.go | 2 +- images/virtualization-artifact/pkg/controller/vmop/gc.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/gc.go b/images/virtualization-artifact/pkg/controller/vm/gc.go index 7d42a6ae63..8c84b3c91a 100644 --- a/images/virtualization-artifact/pkg/controller/vm/gc.go +++ b/images/virtualization-artifact/pkg/controller/vm/gc.go @@ -129,7 +129,7 @@ func (m *vmimGCManager) getIndex(obj client.Object) string { if !ok { return "" } - return types.NamespacedName{Namespace: vmim.GetNamespace(), Name: vmim.GetName()}.String() + return types.NamespacedName{Namespace: vmim.GetNamespace(), Name: vmim.Spec.VMIName}.String() } func vmiMigrationIsFinal(migration *virtv1.VirtualMachineInstanceMigration) bool { diff --git a/images/virtualization-artifact/pkg/controller/vmop/gc.go b/images/virtualization-artifact/pkg/controller/vmop/gc.go index b2192887a6..9a3917dea5 100644 --- a/images/virtualization-artifact/pkg/controller/vmop/gc.go +++ b/images/virtualization-artifact/pkg/controller/vmop/gc.go @@ -108,5 +108,5 @@ func (m *vmopGCManager) getIndex(obj client.Object) string { if !ok { return "" } - return types.NamespacedName{Namespace: vmop.GetNamespace(), Name: vmop.GetName()}.String() + return types.NamespacedName{Namespace: vmop.GetNamespace(), Name: vmop.Spec.VirtualMachine}.String() } From 9796c8276922f82606e95b3a71e5f396dc7a06a3 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Mon, 8 Sep 2025 10:17:27 +0300 Subject: [PATCH 3/3] fix Signed-off-by: Yaroslav Borbat --- .../pkg/config/load_gc_settings.go | 3 +-- .../pkg/controller/gc/filter.go | 15 +++++++++++---- .../pkg/controller/vm/gc.go | 17 +++++++---------- .../virtualization-controller/_helpers.tpl | 2 -- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/images/virtualization-artifact/pkg/config/load_gc_settings.go b/images/virtualization-artifact/pkg/config/load_gc_settings.go index 343b71101d..f218ae4fee 100644 --- a/images/virtualization-artifact/pkg/config/load_gc_settings.go +++ b/images/virtualization-artifact/pkg/config/load_gc_settings.go @@ -29,7 +29,6 @@ const ( GcVmopScheduleVar = "GC_VMOP_SCHEDULE" GcVMIMigrationTTLVar = "GC_VMI_MIGRATION_TTL" GcVMIMigrationScheduleVar = "GC_VMI_MIGRATION_SCHEDULE" - GcCompletedPodTTLVar = "GC_COMPLETED_POD_TTL" GcCompletedPodScheduleVar = "GC_COMPLETED_POD_SCHEDULE" ) @@ -58,7 +57,7 @@ func LoadGcSettings() (GCSettings, error) { } gcSettings.VMIMigration = base - base, err = GetBaseGCSettingsFromEnv(GcCompletedPodScheduleVar, GcCompletedPodTTLVar) + base, err = GetBaseGCSettingsFromEnv(GcCompletedPodScheduleVar, "") if err != nil { return gcSettings, err } diff --git a/images/virtualization-artifact/pkg/controller/gc/filter.go b/images/virtualization-artifact/pkg/controller/gc/filter.go index 75f5aabc69..1a7d529343 100644 --- a/images/virtualization-artifact/pkg/controller/gc/filter.go +++ b/images/virtualization-artifact/pkg/controller/gc/filter.go @@ -56,17 +56,24 @@ func DefaultFilter(objs []client.Object, isCandidate IsCandidate, ttl time.Durat } result := expired + result = append(result, KeepLastRemoveOld(nonExpired, indexFunc, maxCount, now)...) + + return result +} + +func KeepLastRemoveOld(objs []client.Object, indexFunc IndexFunc, maxCount int, now time.Time) []client.Object { if maxCount <= 0 { - return result + return nil } - slices.SortFunc(nonExpired, func(a, b client.Object) int { + slices.SortFunc(objs, func(a, b client.Object) int { return cmp.Compare(getAge(a, now), getAge(b, now)) }) - // Keep maxCount first items (most recently created) for each common index. Index example: virtual machine name. + var result []client.Object + indexed := make(map[string]int) - for _, obj := range nonExpired { + for _, obj := range objs { index := indexFunc(obj) count := indexed[index] if count >= maxCount { diff --git a/images/virtualization-artifact/pkg/controller/vm/gc.go b/images/virtualization-artifact/pkg/controller/vm/gc.go index 8c84b3c91a..e2ff8110f4 100644 --- a/images/virtualization-artifact/pkg/controller/vm/gc.go +++ b/images/virtualization-artifact/pkg/controller/vm/gc.go @@ -62,7 +62,7 @@ func SetupGCCompletedPods( log *log.Logger, gcSettings config.BaseGcSettings, ) error { - podGCMgr := newCompletedPodGCManager(mgr.GetClient(), gcSettings.TTL.Duration, 10) + podGCMgr := newCompletedPodGCManager(mgr.GetClient(), 10) source, err := gc.NewCronSource(gcSettings.Schedule, podGCMgr, log.With("resource", "pod")) if err != nil { return err @@ -138,20 +138,15 @@ func vmiMigrationIsFinal(migration *virtv1.VirtualMachineInstanceMigration) bool type completedPodGCmanager struct { client client.Client - ttl time.Duration max int } -func newCompletedPodGCManager(client client.Client, ttl time.Duration, max int) *completedPodGCmanager { - if ttl == 0 { - ttl = 24 * time.Hour - } +func newCompletedPodGCManager(client client.Client, max int) *completedPodGCmanager { if max == 0 { max = 10 } return &completedPodGCmanager{ client: client, - ttl: ttl, max: max, } } @@ -178,12 +173,14 @@ func (m *completedPodGCmanager) ListForDelete(ctx context.Context, now time.Time return nil, err } - objs := make([]client.Object, 0, len(podList.Items)) + var objs []client.Object for _, pod := range podList.Items { - objs = append(objs, &pod) + if m.ShouldBeDeleted(&pod) { + objs = append(objs, &pod) + } } - result := gc.DefaultFilter(objs, m.ShouldBeDeleted, m.ttl, m.getIndex, m.max, now) + result := gc.KeepLastRemoveOld(objs, m.getIndex, m.max, now) return result, nil } diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index 6d8ff22b2f..b8e9c079a0 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -78,8 +78,6 @@ true value: "24h" - name: GC_VMI_MIGRATION_SCHEDULE value: "0 0 * * *" -- name: GC_COMPLETED_POD_TTL - value: "24h" - name: GC_COMPLETED_POD_SCHEDULE value: "0 0 * * *" {{- if (hasKey .Values.virtualization.internal.moduleConfig "liveMigration") }}