Set DSCP only on traffic leaving cluster

Author: mazdakn

felix/bpf-gpl/conntrack_types.h

@@ -1,5 +1,5 @@
 // Project Calico BPF dataplane programs.
-// Copyright (c) 2020-2021 Tigera, Inc. All rights reserved.
+// Copyright (c) 2020-2025 Tigera, Inc. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
 
 #ifndef __CALI_CONNTRACK_TYPES_H__
@@ -39,6 +39,7 @@ enum cali_ct_type {
 #define CALI_CT_FLAG_NP_REMOTE	0x1000 /* marks connections from local host to remote backend of a nodeport */
 #define CALI_CT_FLAG_NP_NO_DSR	0x2000 /* marks connections from a client which is excluded from DSR */
 #define CALI_CT_FLAG_SKIP_REDIR_PEER	0x4000 /* marks connections from a client which is excluded from redir */
+#define CALI_CT_FLAG_CLUSTER_EXTERNAL 	0x8000 /* marks connections with source or destination outside cluster */
 
 struct calico_ct_leg {
 	__u64 bytes;

felix/bpf-gpl/qos.h

@@ -108,9 +108,13 @@ static CALI_BPF_INLINE int qos_enforce_packet_rate(struct cali_tc_ctx *ctx)
 	return TC_ACT_SHOT;
 }
 
-static CALI_BPF_INLINE bool qos_set_dscp(struct cali_tc_ctx *ctx)
+static CALI_BPF_INLINE bool qos_dscp_needs_update(struct cali_tc_ctx *ctx)
+{
+	return ((ctx->state->flags & CALI_ST_CLUSTER_EXTERNAL) && EGRESS_DSCP >= 0);
+}
+
+static CALI_BPF_INLINE bool qos_dscp_set(struct cali_tc_ctx *ctx)
 {
-	// TODO (mazdak): set DSCP only if traffic is leaving cluster
 	__s8 dscp = EGRESS_DSCP;
 	CALI_DEBUG("setting dscp to %d", dscp);
 

felix/bpf-gpl/routes.h

@@ -96,6 +96,7 @@ static CALI_BPF_INLINE enum cali_rt_flags cali_rt_lookup_flags(ipv46_addr_t *add
 #define cali_rt_flags_local_tunneled_host(t) (((t) & (CALI_RT_LOCAL | CALI_RT_HOST | CALI_RT_TUNNELED)) == (CALI_RT_LOCAL | CALI_RT_HOST | CALI_RT_TUNNELED))
 #define cali_rt_flags_is_in_pool(t) (((t) & CALI_RT_IN_POOL) == CALI_RT_IN_POOL)
 #define cali_rt_flags_skip_ingress_redirect(t) (((t) & CALI_RT_SKIP_INGRESS_REDIRECT))
+#define cali_rt_flags_external(t) (!((t) & (CALI_RT_WORKLOAD | CALI_RT_HOST)))
 
 static CALI_BPF_INLINE bool rt_addr_is_local_host(ipv46_addr_t *addr)
 {
@@ -117,6 +118,17 @@ static CALI_BPF_INLINE bool rt_addr_is_local_tunneled_host(ipv46_addr_t *addr)
 	return cali_rt_flags_local_tunneled_host(cali_rt_lookup_flags(addr));
 }
 
+static CALI_BPF_INLINE bool rt_addr_is_external(ipv46_addr_t *addr)
+{
+	return cali_rt_flags_external(cali_rt_lookup_flags(addr));
+}
+
+static CALI_BPF_INLINE bool rt_addr_is_host_or_in_pool(ipv46_addr_t *addr)
+{
+	__u32 flags = cali_rt_lookup_flags(addr);
+	return cali_rt_flags_host(flags) || cali_rt_flags_is_in_pool(flags);
+}
+
 // Don't perform SNAT if either:
 // - packet is destined to an address in an IP pool;
 // - packet is destined to local host; or

felix/bpf-gpl/tc.c

@@ -366,6 +366,9 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx)
 	if (ctx->state->ct_result.flags & CALI_CT_FLAG_NAT_OUT) {
 		ctx->state->flags |= CALI_ST_NAT_OUTGOING;
 	}
+	if (ctx->state->ct_result.flags & CALI_CT_FLAG_CLUSTER_EXTERNAL) {
+		ctx->state->flags |= CALI_ST_CLUSTER_EXTERNAL;
+	}
 
 	if (CALI_F_TO_HOST && !CALI_F_NAT_IF &&
 			(ct_result_rc(ctx->state->ct_result.rc) == CALI_CT_ESTABLISHED ||
@@ -545,17 +548,33 @@ static CALI_BPF_INLINE void calico_tc_process_ct_lookup(struct cali_tc_ctx *ctx)
 				ctx->state->flags |= CALI_ST_NAT_OUTGOING;
 			}
 		}
+		// Check if traffic is leaving cluster. We might need to set DSCP later.
+		if (cali_rt_flags_is_in_pool(r->flags) && rt_addr_is_external(&ctx->state->post_nat_ip_dst)) {
+			CALI_DEBUG("Outside cluster dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst));
+			ctx->state->flags |= CALI_ST_CLUSTER_EXTERNAL;
+		}
 		/* If 3rd party CNI is used and dest is outside cluster. See commit fc711b192f for details. */
-		if (!(r->flags & CALI_RT_IN_POOL)) {
+		if (!(cali_rt_flags_is_in_pool(r->flags))) {
 			CALI_DEBUG("Source " IP_FMT " not in IP pool", debug_ip(ctx->state->ip_src));
-			r = cali_rt_lookup(&ctx->state->post_nat_ip_dst);
-			if (!r || !(r->flags & (CALI_RT_WORKLOAD | CALI_RT_HOST))) {
+			if (rt_addr_is_external(&ctx->state->post_nat_ip_dst)) {
 				CALI_DEBUG("Outside cluster dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst));
 				ctx->state->flags |= CALI_ST_SKIP_FIB;
 			}
 		}
 	}
 
+	// If either source or destination is outside cluster, set flag as might need to update DSCP later.
+	if ((CALI_F_TO_HEP) && (rt_addr_is_local_host(&ctx->state->ip_src)) &&
+		(rt_addr_is_external(&ctx->state->post_nat_ip_dst))) {
+		CALI_DEBUG("Outside cluster dest " IP_FMT "", debug_ip(ctx->state->post_nat_ip_dst));
+		ctx->state->flags |= CALI_ST_CLUSTER_EXTERNAL;
+	}
+	if ((CALI_F_FROM_HEP) && (rt_addr_is_host_or_in_pool(&ctx->state->post_nat_ip_dst)) &&
+		(rt_addr_is_external(&ctx->state->ip_src))) {
+		CALI_DEBUG("Outside cluster source " IP_FMT "", debug_ip(ctx->state->ip_src));
+		ctx->state->flags |= CALI_ST_CLUSTER_EXTERNAL;
+	}
+
 	/* [SMC] I had to add this revalidation when refactoring the conntrack code to use the context and
 	 * adding possible packet pulls in the VXLAN logic.  I believe it is spurious but the verifier is
 	 * not clever enough to spot that we'd have already bailed out if one of the pulls failed. */
@@ -1327,7 +1346,7 @@ int calico_tc_skb_accepted_entrypoint(struct __sk_buff *skb)
 		deny_reason(ctx, CALI_REASON_DROPPED_BY_QOS);
 		goto deny;
 	}
-	if ((CALI_F_FROM_WEP || CALI_F_TO_HEP) && EGRESS_DSCP >= 0 && !qos_set_dscp(ctx)) {
+	if ((CALI_F_FROM_WEP || CALI_F_TO_HEP) && qos_dscp_needs_update(ctx) && !qos_dscp_set(ctx)) {
 		goto deny;
 	}
 	ctx->fwd = calico_tc_skb_accepted(ctx);
@@ -1408,6 +1427,9 @@ int calico_tc_skb_new_flow_entrypoint(struct __sk_buff *skb)
 	if (state->flags & CALI_ST_NAT_OUTGOING) {
 		ct_ctx_nat->flags |= CALI_CT_FLAG_NAT_OUT;
 	}
+	if (state->flags & CALI_ST_CLUSTER_EXTERNAL) {
+		ct_ctx_nat->flags |= CALI_CT_FLAG_CLUSTER_EXTERNAL;
+	}
 	if (CALI_F_TO_HOST && state->flags & CALI_ST_SKIP_FIB) {
 		ct_ctx_nat->flags |= CALI_CT_FLAG_SKIP_FIB;
 	}

felix/bpf-gpl/types.h

@@ -158,6 +158,9 @@ enum cali_state_flags {
 	CALI_ST_SKIP_REDIR_PEER	  = 0x800,
 	/* CALI_ST_SKIP_REDIR_ONCE skips redirection once for this particular packet */
 	CALI_ST_SKIP_REDIR_ONCE   = 0x1000,
+	/* CALI_ST_CLUSTER_EXTERNAL is set if the packet is heading toward or originating from 
+	 * an endpoint outside the cluster */
+	CALI_ST_CLUSTER_EXTERNAL   = 0x2000,
 };
 
 struct fwd {