WIP L3 NetLB experiment

Author: TortillaZHawaii

pkg/flags/flags.go

@@ -148,6 +148,7 @@ var F = struct {
 	ManageL4LBLogging                         bool
 	EnableNEGsForIngress                      bool
 	EnableIPv6OnlyL4                          bool
+	EnableL3NetLBOptIn                        bool
 	L4ILBLegacyHeadStartTime                  time.Duration
 
 	// ===============================
@@ -361,6 +362,7 @@ L7 load balancing. CSV values accepted. Example: -node-port-ranges=80,8080,400-5
 	flag.BoolVar(&F.ReadOnlyMode, "read-only-controllers", false, "When enabled, this flag runs the IG, NEG, L4 ILB, and L4 NetLB controllers in a read-only mode. This prevents them from executing any mutating API calls (e.g., create, update, delete), allowing you to safely observe controller behavior without modifying resources. The Ingress controller is exempt from this mode.")
 	flag.BoolVar(&F.EnableNEGsForIngress, "enable-negs-for-ingress", true, "Allow the NEG controller to create NEGs for Ingress services.")
 	flag.BoolVar(&F.EnableIPv6OnlyL4, "enable-ipv6-only-l4", false, "Enables IPv6-only mode for the L4 ILB and NetLB controllers, disabling all IPv4-related logic and resource management.")
+	flag.BoolVar(&F.EnableL3NetLBOptIn, "enable-l3-netlb-opt-in", false, "Enables L3 opt in mode for NetLB, where it uses L3 Forwarding Rule and Backend Service regardless of the service spec.")
 	flag.DurationVar(&F.L4ILBLegacyHeadStartTime, "prevent-legacy-race-l4-ilb", 0*time.Second, "Delay before processing new L4 ILB services without existing finalizers. This gives the legacy controller a head start to claim the service, preventing a race condition upon service creation.")
 }
 

pkg/forwardingrules/equal.go

@@ -23,7 +23,7 @@ func EqualIPv4(fr1, fr2 *composite.ForwardingRule) (bool, error) {
 		return false, fmt.Errorf("Equal(): failed to parse backend resource URL from wanted FR, err - %w", err)
 	}
 	return fr1.IPAddress == fr2.IPAddress &&
-		strings.ToLower(fr1.IPProtocol) == strings.ToLower(fr2.IPProtocol) &&
+		strings.EqualFold(fr1.IPProtocol, fr2.IPProtocol) &&
 		fr1.LoadBalancingScheme == fr2.LoadBalancingScheme &&
 		equalPorts(fr1.Ports, fr2.Ports, fr1.PortRange, fr2.PortRange) &&
 		utils.EqualCloudResourceIDs(id1, id2) &&
@@ -45,7 +45,7 @@ func EqualIPv6(fr1, fr2 *composite.ForwardingRule) (bool, error) {
 	if err != nil {
 		return false, fmt.Errorf("EqualIPv6(): failed to parse resource URL from FR, err - %w", err)
 	}
-	return strings.ToLower(fr1.IPProtocol) == strings.ToLower(fr2.IPProtocol) &&
+	return strings.EqualFold(fr1.IPProtocol, fr2.IPProtocol) &&
 		fr1.LoadBalancingScheme == fr2.LoadBalancingScheme &&
 		equalPorts(fr1.Ports, fr2.Ports, fr1.PortRange, fr2.PortRange) &&
 		utils.EqualCloudResourceIDs(id1, id2) &&

pkg/loadbalancers/forwarding_rules.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/ingress-gce/pkg/events"
 	"k8s.io/ingress-gce/pkg/flags"
 	"k8s.io/ingress-gce/pkg/forwardingrules"
+	"k8s.io/ingress-gce/pkg/loadbalancers/l3"
 	"k8s.io/ingress-gce/pkg/translator"
 	"k8s.io/ingress-gce/pkg/utils"
 	"k8s.io/ingress-gce/pkg/utils/namer"
@@ -407,6 +408,12 @@ func (l4netlb *L4NetLB) ensureIPv4ForwardingRule(bsLink string) (*composite.Forw
 		newFwdRule.PortRange = ""
 	}
 
+	if l3.Wants(l4netlb.Service) {
+		newFwdRule.Ports, newFwdRule.PortRange = nil, ""
+		newFwdRule.AllPorts = true
+		newFwdRule.IPProtocol = forwardingrules.ProtocolL3
+	}
+
 	if existingFwdRule != nil {
 		if existingFwdRule.NetworkTier != newFwdRule.NetworkTier {
 			resource := fmt.Sprintf("Forwarding rule (%v)", frName)

pkg/loadbalancers/forwarding_rules_ipv6.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/ingress-gce/pkg/events"
 	"k8s.io/ingress-gce/pkg/flags"
 	"k8s.io/ingress-gce/pkg/forwardingrules"
+	"k8s.io/ingress-gce/pkg/loadbalancers/l3"
 	"k8s.io/ingress-gce/pkg/utils"
 )
 
@@ -291,6 +292,11 @@ func (l4netlb *L4NetLB) buildExpectedIPv6ForwardingRule(bsLink, ipv6AddressToUse
 		fr.Ports = utils.GetPorts(svcPorts)
 		fr.PortRange = ""
 	}
+	if l3.Wants(l4netlb.Service) {
+		fr.Ports, fr.PortRange = nil, ""
+		fr.AllPorts = true
+		fr.IPProtocol = forwardingrules.ProtocolL3
+	}
 
 	return fr, nil
 }

pkg/loadbalancers/l3/wants.go

@@ -0,0 +1,26 @@
+package l3
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/ingress-gce/pkg/flags"
+)
+
+const ExperimentAnnotation = "networking.gke.io/l3-experiment"
+
+func Wants(svc *corev1.Service) bool {
+	if !flags.F.EnableL3NetLBOptIn {
+		return false
+	}
+
+	acceptableValues := map[string]struct{}{
+		"true": {}, "enabled": {}, "enable": {},
+		"on": {}, "yes": {}, "True": {},
+	}
+
+	val, ok := svc.Annotations[ExperimentAnnotation]
+	if !ok {
+		return false
+	}
+	_, ok = acceptableValues[val]
+	return ok
+}

pkg/loadbalancers/l4netlb.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/ingress-gce/pkg/forwardingrules"
 	"k8s.io/ingress-gce/pkg/healthchecksl4"
 	"k8s.io/ingress-gce/pkg/l4lb/metrics"
+	"k8s.io/ingress-gce/pkg/loadbalancers/l3"
 	"k8s.io/ingress-gce/pkg/network"
 	"k8s.io/ingress-gce/pkg/utils"
 	"k8s.io/ingress-gce/pkg/utils/namer"
@@ -353,11 +354,6 @@ func (l4netlb *L4NetLB) provideBackendService(syncResult *L4NetLBSyncResult, hcL
 	bsName := l4netlb.namer.L4Backend(l4netlb.Service.Namespace, l4netlb.Service.Name)
 	servicePorts := l4netlb.Service.Spec.Ports
 
-	protocol := string(utils.GetProtocol(servicePorts))
-	if l4netlb.enableMixedProtocol {
-		protocol = backends.GetProtocol(servicePorts)
-	}
-
 	localityLbPolicy := l4netlb.determineBackendServiceLocalityPolicy()
 
 	connectionTrackingPolicy := l4netlb.connectionTrackingPolicy()
@@ -376,7 +372,7 @@ func (l4netlb *L4NetLB) provideBackendService(syncResult *L4NetLBSyncResult, hcL
 	backendParams := backends.L4BackendServiceParams{
 		Name:                     bsName,
 		HealthCheckLink:          hcLink,
-		Protocol:                 protocol,
+		Protocol:                 l4netlb.backendProtocol(servicePorts),
 		SessionAffinity:          string(l4netlb.Service.Spec.SessionAffinity),
 		Scheme:                   string(cloud.SchemeExternal),
 		NamespacedName:           l4netlb.NamespacedName,
@@ -405,6 +401,17 @@ func (l4netlb *L4NetLB) provideBackendService(syncResult *L4NetLBSyncResult, hcL
 	return bs.SelfLink
 }
 
+func (l4netlb *L4NetLB) backendProtocol(servicePorts []corev1.ServicePort) string {
+	switch {
+	case l3.Wants(l4netlb.Service):
+		return backends.ProtocolL3
+	case l4netlb.enableMixedProtocol:
+		return backends.GetProtocol(servicePorts)
+	default:
+		return string(utils.GetProtocol(servicePorts))
+	}
+}
+
 func (l4netlb *L4NetLB) ensureDualStackResources(result *L4NetLBSyncResult, nodeNames []string, bsLink string) {
 	if utils.NeedsIPv4(l4netlb.Service) {
 		l4netlb.ensureIPv4Resources(result, nodeNames, bsLink)
@@ -422,7 +429,7 @@ func (l4netlb *L4NetLB) ensureDualStackResources(result *L4NetLBSyncResult, node
 // - IPv4 Forwarding Rule
 // - IPv4 Firewall
 func (l4netlb *L4NetLB) ensureIPv4Resources(result *L4NetLBSyncResult, nodeNames []string, bsLink string) {
-	if l4netlb.enableMixedProtocol && forwardingrules.NeedsMixed(l4netlb.Service.Spec.Ports) {
+	if !l3.Wants(l4netlb.Service) && l4netlb.enableMixedProtocol && forwardingrules.NeedsMixed(l4netlb.Service.Spec.Ports) {
 		l4netlb.ensureIPv4MixedResources(result, nodeNames, bsLink)
 		return
 	}
@@ -436,11 +443,7 @@ func (l4netlb *L4NetLB) ensureIPv4Resources(result *L4NetLBSyncResult, nodeNames
 		result.MetricsLegacyState.IsUserError = IsUserError(err)
 		return
 	}
-	if fr.IPProtocol == string(corev1.ProtocolTCP) {
-		result.Annotations[annotations.TCPForwardingRuleKey] = fr.Name
-	} else {
-		result.Annotations[annotations.UDPForwardingRuleKey] = fr.Name
-	}
+	result.Annotations[forwardingRuleAnnotationKey(fr)] = fr.Name
 	result.MetricsLegacyState.IsManagedIP = ipAddrType == address.IPAddrManaged
 	result.MetricsLegacyState.IsPremiumTier = fr.NetworkTier == cloud.NetworkTierPremium.ToGCEValue()
 
@@ -453,6 +456,33 @@ func (l4netlb *L4NetLB) ensureIPv4Resources(result *L4NetLBSyncResult, nodeNames
 	result.Status = utils.AddIPToLBStatus(result.Status, fr.IPAddress)
 }
 
+func forwardingRuleAnnotationKey(fr *composite.ForwardingRule) string {
+	m := map[string]map[string]string{
+		"IPV4": {
+			forwardingrules.ProtocolL3:  annotations.L3ForwardingRuleKey,
+			forwardingrules.ProtocolTCP: annotations.TCPForwardingRuleKey,
+			forwardingrules.ProtocolUDP: annotations.UDPForwardingRuleKey,
+		},
+		"IPV6": {
+			forwardingrules.ProtocolL3:  annotations.L3ForwardingRuleIPv6Key,
+			forwardingrules.ProtocolTCP: annotations.TCPForwardingRuleIPv6Key,
+			forwardingrules.ProtocolUDP: annotations.UDPForwardingRuleIPv6Key,
+		},
+	}
+
+	version := fr.IpVersion
+	if version == "" {
+		version = "IPV4"
+	}
+
+	protocol := fr.IPProtocol
+	if protocol == "" {
+		protocol = "UDP"
+	}
+
+	return m[version][protocol]
+}
+
 func (l4netlb *L4NetLB) ensureIPv4MixedResources(result *L4NetLBSyncResult, nodeNames []string, bsLink string) {
 	res, err := l4netlb.mixedManager.EnsureIPv4(bsLink)
 

pkg/loadbalancers/l4netlbipv6.go

@@ -24,7 +24,6 @@ import (
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
 	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
 	compute "google.golang.org/api/compute/v1"
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/ingress-gce/pkg/annotations"
 	"k8s.io/ingress-gce/pkg/firewalls"
 	"k8s.io/ingress-gce/pkg/utils"
@@ -50,11 +49,7 @@ func (l4netlb *L4NetLB) ensureIPv6Resources(syncResult *L4NetLBSyncResult, nodeN
 		return
 	}
 
-	if ipv6fr.IPProtocol == string(corev1.ProtocolTCP) {
-		syncResult.Annotations[annotations.TCPForwardingRuleIPv6Key] = ipv6fr.Name
-	} else {
-		syncResult.Annotations[annotations.UDPForwardingRuleIPv6Key] = ipv6fr.Name
-	}
+	syncResult.Annotations[forwardingRuleAnnotationKey(ipv6fr)] = ipv6fr.Name
 
 	// Google Cloud creates ipv6 forwarding rules with IPAddress in CIDR form. We will take only first address
 	trimmedIPv6Address := strings.Split(ipv6fr.IPAddress, "/")[0]
@@ -120,6 +115,15 @@ func (l4netlb *L4NetLB) ensureIPv6NodesFirewall(ipAddress string, nodeNames []st
 	svcPorts := l4netlb.Service.Spec.Ports
 	portRanges := utils.GetServicePortRanges(svcPorts)
 	protocol := utils.GetProtocol(svcPorts)
+	allowed := []*compute.FirewallAllowed{
+		{
+			IPProtocol: string(protocol),
+			Ports:      portRanges,
+		},
+	}
+	if l4netlb.enableMixedProtocol {
+		allowed = firewalls.AllowedForService(svcPorts)
+	}
 
 	fwLogger := l4netlb.svcLogger.WithValues("firewallName", firewallName)
 	fwLogger.V(2).Info("Ensuring IPv6 nodes firewall for L4 NetLB Service", "ipAddress", ipAddress, "protocol", protocol, "len(nodeNames)", len(nodeNames), "portRanges", portRanges)
@@ -135,12 +139,7 @@ func (l4netlb *L4NetLB) ensureIPv6NodesFirewall(ipAddress string, nodeNames []st
 	}
 
 	ipv6nodesFWRParams := firewalls.FirewallParams{
-		Allowed: []*compute.FirewallAllowed{
-			{
-				IPProtocol: string(protocol),
-				Ports:      portRanges,
-			},
-		},
+		Allowed:           allowed,
 		SourceRanges:      ipv6SourceRanges,
 		DestinationRanges: []string{ipAddress},
 		Name:              firewallName,