Change kube-adaptor Leader Election Loss Error Handling (#2296)

* Change kube-adaptor Leader Election Loss Error Handling

Updates the kube-adaptor so that when the skupper-site-leader Lease is
lost the kube-adaptor will retry instead of exiting.

Signed-off-by: Christian Kruse <[email protected]>

* remove harmful lease owner assignment

Signed-off-by: Christian Kruse <[email protected]>

* fix kube flow controller go routine leak

Signed-off-by: Christian Kruse <[email protected]>

---------

Signed-off-by: Christian Kruse <[email protected]>

Author: c-kruse

internal/kube/adaptor/collector.go

@@ -6,8 +6,10 @@ import (
 	"log"
 	"log/slog"
 	"os"
+	"sync"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	"github.com/skupperproject/skupper/api/types"
 	"github.com/skupperproject/skupper/internal/config"
 	"github.com/skupperproject/skupper/internal/flow"
@@ -47,20 +49,6 @@ func (s *StatusSyncClient) Update(ctx context.Context, latest *corev1.ConfigMap)
 	return err
 }
 
-func updateLockOwner(lockname, namespace string, owner *metav1.OwnerReference, cli *internalclient.KubeClient) error {
-	current, err := cli.Kube.CoordinationV1().Leases(namespace).Get(context.TODO(), lockname, metav1.GetOptions{})
-	if err != nil {
-		return err
-	}
-	if owner != nil {
-		current.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
-			*owner,
-		}
-	}
-	_, err = cli.Kube.CoordinationV1().Leases(namespace).Update(context.TODO(), current, metav1.UpdateOptions{})
-	return err
-}
-
 func siteCollector(ctx context.Context, cli *internalclient.KubeClient) {
 	siteData := map[string]string{}
 	platform := config.GetPlatform()
@@ -84,11 +72,6 @@ func siteCollector(ctx context.Context, cli *internalclient.KubeClient) {
 		log.Fatal("Failed to create site status config map ", err.Error())
 	}
 
-	err = updateLockOwner(types.SiteLeaderLockName, cli.Namespace, &owner, cli)
-	if err != nil {
-		log.Println("Update lock error", err.Error())
-	}
-
 	factory := session.NewContainerFactory("amqp://localhost:5672", session.ContainerConfig{ContainerID: "kube-flow-collector"})
 	statusSyncClient := &StatusSyncClient{
 		client: cli.Kube.CoreV1().ConfigMaps(cli.Namespace),
@@ -136,37 +119,56 @@ func startFlowController(ctx context.Context, cli *internalclient.KubeClient) er
 }
 
 func runLeaderElection(lock *resourcelock.LeaseLock, id string, cli *internalclient.KubeClient) {
-	ctx := context.Background()
-	begin := time.Now()
-	podname, _ := os.Hostname()
-	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
-		Lock:            lock,
-		ReleaseOnCancel: true,
-		LeaseDuration:   15 * time.Second,
-		RenewDeadline:   10 * time.Second,
-		RetryPeriod:     2 * time.Second,
-		Callbacks: leaderelection.LeaderCallbacks{
-			OnStartedLeading: func(c context.Context) {
-				log.Printf("COLLECTOR: Leader %s starting site collection after %s\n", podname, time.Since(begin))
-				siteCollector(ctx, cli)
-				if err := startFlowController(ctx, cli); err != nil {
-					log.Printf("COLLECTOR: Failed to start controller for emitting site events: %s", err)
-				}
-			},
-			OnStoppedLeading: func() {
-				// we held the lock but lost it. This indicates that something
-				// went wrong. Exit and restart.
-				log.Fatalf("COLLECTOR: Lost leader lock after %s", time.Since(begin))
-			},
-			OnNewLeader: func(current_id string) {
-				if current_id == id {
-					// Remain as the leader
-					return
-				}
-				log.Printf("COLLECTOR: New leader for site collection is %s\n", current_id)
+	var (
+		mu              sync.Mutex
+		leaderCtx       context.Context
+		leaderCtxCancel func()
+	)
+	// attempt to run leader election forever
+	strategy := backoff.NewExponentialBackOff(backoff.WithMaxElapsedTime(0))
+	backoff.RetryNotify(func() error {
+		leaderelection.RunOrDie(context.Background(), leaderelection.LeaderElectionConfig{
+			Lock:            lock,
+			ReleaseOnCancel: true,
+			LeaseDuration:   15 * time.Second,
+			RenewDeadline:   10 * time.Second,
+			RetryPeriod:     2 * time.Second,
+			Callbacks: leaderelection.LeaderCallbacks{
+				OnStartedLeading: func(ctx context.Context) {
+					mu.Lock()
+					defer mu.Unlock()
+					leaderCtx, leaderCtxCancel = context.WithCancel(ctx)
+					log.Printf("COLLECTOR: Became leader. Starting status sync and site controller after %s.", strategy.GetElapsedTime())
+					siteCollector(leaderCtx, cli)
+					if err := startFlowController(leaderCtx, cli); err != nil {
+						log.Printf("COLLECTOR: Failed to start controller for emitting site events: %s", err)
+					}
+				},
+				OnStoppedLeading: func() {
+					log.Printf("COLLECTOR: Lost leader lock after %s. Stopping status sync and site controller.", strategy.GetElapsedTime())
+					mu.Lock()
+					defer mu.Unlock()
+					if leaderCtxCancel == nil {
+						return
+					}
+					leaderCtxCancel()
+					leaderCtx, leaderCtxCancel = nil, nil
+				},
+				OnNewLeader: func(current_id string) {
+					if current_id == id {
+						// Remain as the leader
+						return
+					}
+					log.Printf("COLLECTOR: New leader for site collection is %s\n", current_id)
+				},
 			},
-		},
-	})
+		})
+		return fmt.Errorf("leader election died")
+	},
+		strategy,
+		func(_ error, d time.Duration) {
+			log.Printf("COLLECTOR: leader election failed. retrying after %s", d)
+		})
 }
 
 func StartCollector(cli *internalclient.KubeClient) {

internal/kube/flow/controller.go

@@ -130,26 +130,26 @@ func (c *Controller) Run(ctx context.Context) {
 		c.logger.Error("amqp session error", slog.Any("error", err), slog.Bool("retryable", retryable))
 	})
 	go c.manager.Run(mgmtCtx)
+	// handle shutdown though work queue
+	go func() {
+		<-mgmtCtx.Done()
+		c.queue.ShutDown()
+	}()
 	for {
-		select {
-		case <-ctx.Done():
+		item, done := c.queue.Get()
+		if done {
+			c.logger.Info("workqueue shutdown")
 			return
-		default:
-			item, done := c.queue.Get()
-			if done {
-				c.logger.Info("workqueue shutdown")
-				return
-			}
-			evnt := item.(workEvent)
-			switch evnt.Type {
-			case eventTypeProcess:
-				if err := c.handlePodEvent(evnt); err != nil {
-					c.queue.AddRateLimited(evnt)
-				}
+		}
+		evnt := item.(workEvent)
+		switch evnt.Type {
+		case eventTypeProcess:
+			if err := c.handlePodEvent(evnt); err != nil {
+				c.queue.AddRateLimited(evnt)
 			}
-			c.queue.Forget(item)
-			c.queue.Done(item)
 		}
+		c.queue.Forget(item)
+		c.queue.Done(item)
 	}
 }