Add default anti affinity (#810)

jiangpengcheng · freeznet · commit 1ed84f431bcf · 2025-07-24T09:31:29.000+08:00
* [feat] Add default affinities to ensure pods:
1. prefer to be in different zones (soft rule)
2. must not be scheduled on the same node

* add test

* fix npe

* fix vulnerabilities

* fix unittest

* fix unittest

* fix ci

* increase kind nodes in ci

* fix ci

* fix ci

* Add a `DisableDefaultAffinity` field to PodPolicy

* Update doc
diff --git a/.ci/tests/integration/cases/global-and-namespaced-config/manifests.yaml b/.ci/tests/integration/cases/global-and-namespaced-config/manifests.yaml
@@ -16,6 +16,7 @@ spec:
       - persistent://public/default/input-java-topic
     typeClassName: java.lang.String
   pod:
+    disableDefaultAffinity: true
     env:
       - name: "podenv"
         value: "podvalue"
diff --git a/.ci/tests/integration/cases/python-function/manifests.yaml b/.ci/tests/integration/cases/python-function/manifests.yaml
@@ -8,8 +8,7 @@ spec:
   className: exclamation_function.ExclamationFunction
   forwardSourceMessageProperty: true
   maxPendingAsyncRequests: 1000
-  replicas: 1
-  maxReplicas: 1
+  replicas: 3
   logTopic: persistent://public/default/py-function-logs
   input:
     topics:
diff --git a/.ci/tests/kind.yaml b/.ci/tests/kind.yaml
@@ -31,4 +31,10 @@ nodes:
         listenAddress: "127.0.0.1"
         # optional: set the protocol to one of TCP, UDP, SCTP.
         # TCP is the default
-        protocol: TCP
+        protocol: TCP
+
+  - role: worker
+    image: kindest/node:v1.23.17
+
+  - role: worker
+    image: kindest/node:v1.23.17
diff --git a/Makefile b/Makefile
@@ -72,7 +72,7 @@ test-ginkgo: generate fmt vet manifests envtest
 
 .PHONY: envtest
 envtest:
-	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20240320141353-395cfc7486e6
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@v0.0.0-20250630142431-4c2e9cec2954
 
 # Build manager binary
 manager: generate fmt vet
diff --git a/api/compute/v1alpha1/common.go b/api/compute/v1alpha1/common.go
@@ -227,6 +227,8 @@ type PodPolicy struct {
 	Env []corev1.EnvVar `json:"env,omitempty"`
 
 	Liveness *Liveness `json:"liveness,omitempty"`
+
+	DisableDefaultAffinity bool `json:"disableDefaultAffinity,omitempty"`
 }
 
 type Runtime struct {
diff --git a/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-backendconfigs.yaml b/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-backendconfigs.yaml
@@ -828,6 +828,8 @@ spec:
                       items:
                         type: string
                       type: array
+                    disableDefaultAffinity:
+                      type: boolean
                     env:
                       items:
                         properties:
diff --git a/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functionmeshes.yaml b/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functionmeshes.yaml
@@ -1127,6 +1127,8 @@ spec:
                             items:
                               type: string
                             type: array
+                          disableDefaultAffinity:
+                            type: boolean
                           env:
                             items:
                               properties:
@@ -5004,6 +5006,8 @@ spec:
                             items:
                               type: string
                             type: array
+                          disableDefaultAffinity:
+                            type: boolean
                           env:
                             items:
                               properties:
@@ -8662,6 +8666,8 @@ spec:
                             items:
                               type: string
                             type: array
+                          disableDefaultAffinity:
+                            type: boolean
                           env:
                             items:
                               properties:
diff --git a/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functions.yaml b/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-functions.yaml
@@ -1146,6 +1146,8 @@ spec:
                       items:
                         type: string
                       type: array
+                    disableDefaultAffinity:
+                      type: boolean
                     env:
                       items:
                         properties:
diff --git a/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-sinks.yaml b/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-sinks.yaml
@@ -1067,6 +1067,8 @@ spec:
                       items:
                         type: string
                       type: array
+                    disableDefaultAffinity:
+                      type: boolean
                     env:
                       items:
                         properties:
diff --git a/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-sources.yaml b/charts/function-mesh-operator/charts/admission-webhook/templates/crd-compute.functionmesh.io-sources.yaml
@@ -1062,6 +1062,8 @@ spec:
                       items:
                         type: string
                       type: array
+                    disableDefaultAffinity:
+                      type: boolean
                     env:
                       items:
                         properties:
diff --git a/charts/function-mesh-operator/templates/controller-manager-deployment.yaml b/charts/function-mesh-operator/templates/controller-manager-deployment.yaml
@@ -65,6 +65,7 @@ spec:
           - --global-backend-config={{ .Values.controllerManager.globalBackendConfig }}
           - --global-backend-config-namespace={{ .Values.controllerManager.globalBackendConfigNamespace }}
           - --namespaced-backend-config={{ .Values.controllerManager.namespacedBackendConfig }}
+          - --add-default-affinity={{ .Values.controllerManager.addDefaultAffinity }}
         env:
           - name: NAMESPACE
             valueFrom:
diff --git a/charts/function-mesh-operator/values.yaml b/charts/function-mesh-operator/values.yaml
@@ -73,6 +73,7 @@ controllerManager:
   globalBackendConfig: global-backend-config
   globalBackendConfigNamespace: default
   namespacedBackendConfig: backend-config
+  addDefaultAffinity: true
 
 admissionWebhook:
   enabled: true
diff --git a/config/crd/bases/compute.functionmesh.io_backendconfigs.yaml b/config/crd/bases/compute.functionmesh.io_backendconfigs.yaml
@@ -806,6 +806,8 @@ spec:
                     items:
                       type: string
                     type: array
+                  disableDefaultAffinity:
+                    type: boolean
                   env:
                     items:
                       properties:
diff --git a/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml b/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml
@@ -1127,6 +1127,8 @@ spec:
                           items:
                             type: string
                           type: array
+                        disableDefaultAffinity:
+                          type: boolean
                         env:
                           items:
                             properties:
@@ -5004,6 +5006,8 @@ spec:
                           items:
                             type: string
                           type: array
+                        disableDefaultAffinity:
+                          type: boolean
                         env:
                           items:
                             properties:
@@ -8662,6 +8666,8 @@ spec:
                           items:
                             type: string
                           type: array
+                        disableDefaultAffinity:
+                          type: boolean
                         env:
                           items:
                             properties:
diff --git a/config/crd/bases/compute.functionmesh.io_functions.yaml b/config/crd/bases/compute.functionmesh.io_functions.yaml
@@ -1124,6 +1124,8 @@ spec:
                     items:
                       type: string
                     type: array
+                  disableDefaultAffinity:
+                    type: boolean
                   env:
                     items:
                       properties:
diff --git a/config/crd/bases/compute.functionmesh.io_sinks.yaml b/config/crd/bases/compute.functionmesh.io_sinks.yaml
@@ -1045,6 +1045,8 @@ spec:
                     items:
                       type: string
                     type: array
+                  disableDefaultAffinity:
+                    type: boolean
                   env:
                     items:
                       properties:
diff --git a/config/crd/bases/compute.functionmesh.io_sources.yaml b/config/crd/bases/compute.functionmesh.io_sources.yaml
@@ -1040,6 +1040,8 @@ spec:
                     items:
                       type: string
                     type: array
+                  disableDefaultAffinity:
+                    type: boolean
                   env:
                     items:
                       properties:
diff --git a/controllers/spec/common.go b/controllers/spec/common.go
@@ -463,9 +463,10 @@ func makePodTemplate(container *corev1.Container, filebeatContainer *corev1.Cont
 	if filebeatContainer != nil {
 		containers = append(containers, *filebeatContainer)
 	}
+	mergedLabels := mergeMaps(labels, Configs.ResourceLabels, policy.Labels)
 	return &corev1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
-			Labels:      mergeMaps(labels, Configs.ResourceLabels, policy.Labels),
+			Labels:      mergedLabels,
 			Annotations: generateAnnotations(Configs.ResourceAnnotations, policy.Annotations),
 		},
 		Spec: corev1.PodSpec{
@@ -474,7 +475,7 @@ func makePodTemplate(container *corev1.Container, filebeatContainer *corev1.Cont
 			TerminationGracePeriodSeconds: policy.TerminationGracePeriodSeconds,
 			Volumes:                       volumes,
 			NodeSelector:                  policy.NodeSelector,
-			Affinity:                      policy.Affinity,
+			Affinity:                      GenerateAffinity(policy.Affinity, mergedLabels, policy.DisableDefaultAffinity),
 			Tolerations:                   policy.Tolerations,
 			SecurityContext:               podSecurityContext,
 			ImagePullSecrets:              policy.ImagePullSecrets,
@@ -483,6 +484,36 @@ func makePodTemplate(container *corev1.Container, filebeatContainer *corev1.Cont
 	}
 }
 
+func GenerateAffinity(affinity *corev1.Affinity, labels map[string]string, disableDefaultAffinity bool) *corev1.Affinity {
+	if !utils.AddDefaultAffinity || disableDefaultAffinity {
+		return affinity
+	}
+	if affinity == nil {
+		affinity = &corev1.Affinity{}
+	}
+	if affinity.PodAntiAffinity == nil {
+		affinity.PodAntiAffinity = &corev1.PodAntiAffinity{}
+	}
+
+	// add default pod anti-affinity rules to ensure replica pods prefer running on different node
+	if affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution == nil {
+		affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []corev1.WeightedPodAffinityTerm{}
+	}
+	affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(
+		affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution,
+		corev1.WeightedPodAffinityTerm{
+			Weight: 100,
+			PodAffinityTerm: corev1.PodAffinityTerm{
+				LabelSelector: &metav1.LabelSelector{
+					MatchLabels: labels,
+				},
+				TopologyKey: "kubernetes.io/hostname",
+			},
+		},
+	)
+	return affinity
+}
+
 func MakeJavaFunctionCommand(downloadPath, packageFile, name, clusterName, generateLogConfigCommand, logLevel, details, extraDependenciesDir, uid string,
 	memory *resource.Quantity, javaOpts []string, hasPulsarctl, hasWget, authProvided, tlsProvided bool, secretMaps map[string]v1alpha1.SecretRef,
 	state *v1alpha1.Stateful,
diff --git a/main.go b/main.go
@@ -79,6 +79,7 @@ func main() {
 	var globalBackendConfigNamespace string
 	var namespacedBackendConfig string
 	var secureMetrics bool
+	var addDefaultAffinity bool
 	flag.StringVar(&metricsAddr, "metrics-addr", lookupEnvOrString("METRICS_ADDR", ":8443"),
 		"The address the metric endpoint binds to.")
 	flag.StringVar(&leaderElectionID, "leader-election-id",
@@ -112,13 +113,16 @@ func main() {
 		"The backend config name used for functions&sinks&sources in the same namespace")
 	flag.BoolVar(&secureMetrics, "metrics-secure", true, "If set, the metrics endpoint is served securely via HTTPS."+
 		" Use --metrics-secure=false to use HTTP instead.")
+	flag.BoolVar(&addDefaultAffinity, "add-default-affinity", lookupEnvOrBool("ADD_DEFAULT_AFFINITY", true), "If set, the generated pod will add one default podAntiAffinity:"+
+		" make pods prefer not be scheduled on the same node (soft rule).")
 	flag.Parse()
 
 	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
 	utils.EnableInitContainers = enableInitContainers
 	utils.GlobalBackendConfig = globalBackendConfig
 	utils.GlobalBackendConfigNamespace = globalBackendConfigNamespace
 	utils.NamespacedBackendConfig = namespacedBackendConfig
+	utils.AddDefaultAffinity = addDefaultAffinity
 
 	// enable pprof
 	if enablePprof {
diff --git a/utils/configs.go b/utils/configs.go
@@ -22,3 +22,4 @@ var EnableInitContainers = false
 var GlobalBackendConfig = ""
 var GlobalBackendConfigNamespace = "default"
 var NamespacedBackendConfig = ""
+var AddDefaultAffinity = true

Original file line number	Diff line number	Diff line change
`@@ -227,6 +227,8 @@ type PodPolicy struct {`
`227`	`227`	Env []corev1.EnvVar `json:"env,omitempty"`
`228`	`228`
`229`	`229`	Liveness *Liveness `json:"liveness,omitempty"`
	`230`	`+`
	`231`	+ DisableDefaultAffinity bool `json:"disableDefaultAffinity,omitempty"`
`230`	`232`	`}`
`231`	`233`
`232`	`234`	`type Runtime struct {`