diff --git a/images/virtualization-artifact/cmd/virtualization-controller/main.go b/images/virtualization-artifact/cmd/virtualization-controller/main.go index 3609029a02..f80a8e111b 100644 --- a/images/virtualization-artifact/cmd/virtualization-controller/main.go +++ b/images/virtualization-artifact/cmd/virtualization-controller/main.go @@ -39,6 +39,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/deckhouse/deckhouse/pkg/log" + appconfig "github.com/deckhouse/virtualization-controller/pkg/config" "github.com/deckhouse/virtualization-controller/pkg/controller/cvi" "github.com/deckhouse/virtualization-controller/pkg/controller/evacuation" @@ -326,7 +327,15 @@ func main() { log.Error(err.Error()) os.Exit(1) } - if err = vm.SetupGC(mgr, vmLogger, gcSettings.VMIMigration); err != nil { + + gcMIGLogger := logger.NewControllerLogger(vm.GCVMMigrationControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList) + if err = vm.SetupGCMigrations(mgr, gcMIGLogger, gcSettings.VMIMigration); err != nil { + log.Error(err.Error()) + os.Exit(1) + } + + gcPODGLogger := logger.NewControllerLogger(vm.GCCompletedPodControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList) + if err = vm.SetupGCCompletedPods(mgr, gcPODGLogger, gcSettings.CompletedPod); err != nil { log.Error(err.Error()) os.Exit(1) } diff --git a/images/virtualization-artifact/pkg/config/load_gc_settings.go b/images/virtualization-artifact/pkg/config/load_gc_settings.go index 7cac99823f..f218ae4fee 100644 --- a/images/virtualization-artifact/pkg/config/load_gc_settings.go +++ b/images/virtualization-artifact/pkg/config/load_gc_settings.go @@ -29,11 +29,13 @@ const ( GcVmopScheduleVar = "GC_VMOP_SCHEDULE" GcVMIMigrationTTLVar = "GC_VMI_MIGRATION_TTL" GcVMIMigrationScheduleVar = "GC_VMI_MIGRATION_SCHEDULE" + GcCompletedPodScheduleVar = "GC_COMPLETED_POD_SCHEDULE" ) type GCSettings struct { VMOP BaseGcSettings VMIMigration BaseGcSettings + CompletedPod BaseGcSettings } type BaseGcSettings struct { @@ -55,6 +57,12 @@ func LoadGcSettings() (GCSettings, error) { } gcSettings.VMIMigration = base + base, err = GetBaseGCSettingsFromEnv(GcCompletedPodScheduleVar, "") + if err != nil { + return gcSettings, err + } + gcSettings.CompletedPod = base + return gcSettings, nil } diff --git a/images/virtualization-artifact/pkg/controller/gc/filter.go b/images/virtualization-artifact/pkg/controller/gc/filter.go index 75f5aabc69..1a7d529343 100644 --- a/images/virtualization-artifact/pkg/controller/gc/filter.go +++ b/images/virtualization-artifact/pkg/controller/gc/filter.go @@ -56,17 +56,24 @@ func DefaultFilter(objs []client.Object, isCandidate IsCandidate, ttl time.Durat } result := expired + result = append(result, KeepLastRemoveOld(nonExpired, indexFunc, maxCount, now)...) + + return result +} + +func KeepLastRemoveOld(objs []client.Object, indexFunc IndexFunc, maxCount int, now time.Time) []client.Object { if maxCount <= 0 { - return result + return nil } - slices.SortFunc(nonExpired, func(a, b client.Object) int { + slices.SortFunc(objs, func(a, b client.Object) int { return cmp.Compare(getAge(a, now), getAge(b, now)) }) - // Keep maxCount first items (most recently created) for each common index. Index example: virtual machine name. + var result []client.Object + indexed := make(map[string]int) - for _, obj := range nonExpired { + for _, obj := range objs { index := indexFunc(obj) count := indexed[index] if count >= maxCount { diff --git a/images/virtualization-artifact/pkg/controller/vm/gc.go b/images/virtualization-artifact/pkg/controller/vm/gc.go index 5ccdc8bcdd..e2ff8110f4 100644 --- a/images/virtualization-artifact/pkg/controller/vm/gc.go +++ b/images/virtualization-artifact/pkg/controller/vm/gc.go @@ -20,18 +20,25 @@ import ( "context" "time" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/virtualization-controller/pkg/config" "github.com/deckhouse/virtualization-controller/pkg/controller/gc" ) -const gcVMMigrationControllerName = "vmi-migration-gc-controller" +const ( + GCVMMigrationControllerName = "vmi-migration-gc-controller" + GCCompletedPodControllerName = "completed-pod-gc-controller" +) -func SetupGC( +func SetupGCMigrations( mgr manager.Manager, log *log.Logger, gcSettings config.BaseGcSettings, @@ -42,8 +49,7 @@ func SetupGC( if err != nil { return err } - - return gc.SetupGcController(gcVMMigrationControllerName, + return gc.SetupGcController(GCVMMigrationControllerName, mgr, log, source, @@ -51,6 +57,24 @@ func SetupGC( ) } +func SetupGCCompletedPods( + mgr manager.Manager, + log *log.Logger, + gcSettings config.BaseGcSettings, +) error { + podGCMgr := newCompletedPodGCManager(mgr.GetClient(), 10) + source, err := gc.NewCronSource(gcSettings.Schedule, podGCMgr, log.With("resource", "pod")) + if err != nil { + return err + } + return gc.SetupGcController(GCCompletedPodControllerName, + mgr, + log, + source, + podGCMgr, + ) +} + func newVMIMGCManager(client client.Client, ttl time.Duration, max int) *vmimGCManager { if ttl == 0 { ttl = 24 * time.Hour @@ -105,9 +129,70 @@ func (m *vmimGCManager) getIndex(obj client.Object) string { if !ok { return "" } - return vmim.Spec.VMIName + return types.NamespacedName{Namespace: vmim.GetNamespace(), Name: vmim.Spec.VMIName}.String() } func vmiMigrationIsFinal(migration *virtv1.VirtualMachineInstanceMigration) bool { return migration.Status.Phase == virtv1.MigrationFailed || migration.Status.Phase == virtv1.MigrationSucceeded } + +type completedPodGCmanager struct { + client client.Client + max int +} + +func newCompletedPodGCManager(client client.Client, max int) *completedPodGCmanager { + if max == 0 { + max = 10 + } + return &completedPodGCmanager{ + client: client, + max: max, + } +} + +func (m *completedPodGCmanager) New() client.Object { + return &corev1.Pod{} +} + +func (m *completedPodGCmanager) ShouldBeDeleted(obj client.Object) bool { + pod, ok := obj.(*corev1.Pod) + if !ok { + return false + } + + return pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed +} + +func (m *completedPodGCmanager) ListForDelete(ctx context.Context, now time.Time) ([]client.Object, error) { + podList := &corev1.PodList{} + err := m.client.List(ctx, podList, client.MatchingLabels{ + "kubevirt.internal.virtualization.deckhouse.io": "virt-launcher", + }) + if err != nil { + return nil, err + } + + var objs []client.Object + for _, pod := range podList.Items { + if m.ShouldBeDeleted(&pod) { + objs = append(objs, &pod) + } + } + + result := gc.KeepLastRemoveOld(objs, m.getIndex, m.max, now) + + return result, nil +} + +func (m *completedPodGCmanager) getIndex(obj client.Object) string { + pod, ok := obj.(*corev1.Pod) + if !ok { + return "" + } + owner := metav1.GetControllerOf(pod) + if owner != nil { + return string(owner.UID) + } + return "" +} diff --git a/images/virtualization-artifact/pkg/controller/vmop/gc.go b/images/virtualization-artifact/pkg/controller/vmop/gc.go index 567d6977f1..9a3917dea5 100644 --- a/images/virtualization-artifact/pkg/controller/vmop/gc.go +++ b/images/virtualization-artifact/pkg/controller/vmop/gc.go @@ -20,10 +20,12 @@ import ( "context" "time" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/deckhouse/deckhouse/pkg/log" + commonvmop "github.com/deckhouse/virtualization-controller/pkg/common/vmop" "github.com/deckhouse/virtualization-controller/pkg/config" "github.com/deckhouse/virtualization-controller/pkg/controller/gc" @@ -106,5 +108,5 @@ func (m *vmopGCManager) getIndex(obj client.Object) string { if !ok { return "" } - return vmop.Spec.VirtualMachine + return types.NamespacedName{Namespace: vmop.GetNamespace(), Name: vmop.Spec.VirtualMachine}.String() } diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index a8f6abb981..b8e9c079a0 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -78,6 +78,8 @@ true value: "24h" - name: GC_VMI_MIGRATION_SCHEDULE value: "0 0 * * *" +- name: GC_COMPLETED_POD_SCHEDULE + value: "0 0 * * *" {{- if (hasKey .Values.virtualization.internal.moduleConfig "liveMigration") }} - name: LIVE_MIGRATION_BANDWIDTH_PER_NODE value: {{ .Values.virtualization.internal.moduleConfig.liveMigration.bandwidthPerNode | quote }}