Fix kfp-kubernetes-library-tests and backend driver Optional field handling

- Regenerate Python protobuf bindings to include the optional field for SecretAsEnv and ConfigMapAsEnv
- Update test_volume.py to expect optional: False in secretAsEnv
- Fix backend driver to properly handle Optional field for all cases:
  * Not specified: Optional = nil → defaults to false in YAML (backward compatible)
  * Explicitly false: Optional = &false → optional: false in YAML
  * Explicitly true: Optional = &true → optional: true in YAML
- All 110 kfp-kubernetes-library-tests pass
- All backend driver tests pass
- Sample pipeline compilation tests pass with correct optional field handling
- Should fix KFP Samples test failures related to pod-spec-patch
- Fixes PR #12216 test failures

Signed-off-by: ddalvi <[email protected]>

Author: DharmitD

backend/src/v2/driver/k8s.go

@@ -273,12 +273,19 @@ func extendPodSpecPatch(
 	// Get secret env information
 	for _, secretAsEnv := range kubernetesExecutorConfig.GetSecretAsEnv() {
 		for _, keyToEnv := range secretAsEnv.GetKeyToEnv() {
+			secretKeySelector := &k8score.SecretKeySelector{
+				Key: keyToEnv.GetSecretKey(),
+			}
+
+			// Set Optional field when explicitly provided (true or false), leave nil when not specified
+			if secretAsEnv.Optional != nil {
+				secretKeySelector.Optional = secretAsEnv.Optional
+			}
+
 			secretEnvVar := k8score.EnvVar{
 				Name: keyToEnv.GetEnvVar(),
 				ValueFrom: &k8score.EnvVarSource{
-					SecretKeyRef: &k8score.SecretKeySelector{
-						Key: keyToEnv.GetSecretKey(),
-					},
+					SecretKeyRef: secretKeySelector,
 				},
 			}
 
@@ -341,12 +348,19 @@ func extendPodSpecPatch(
 	// Get config map env information
 	for _, configMapAsEnv := range kubernetesExecutorConfig.GetConfigMapAsEnv() {
 		for _, keyToEnv := range configMapAsEnv.GetKeyToEnv() {
+			configMapKeySelector := &k8score.ConfigMapKeySelector{
+				Key: keyToEnv.GetConfigMapKey(),
+			}
+
+			// Set Optional field when explicitly provided (true or false), leave nil when not specified
+			if configMapAsEnv.Optional != nil {
+				configMapKeySelector.Optional = configMapAsEnv.Optional
+			}
+
 			configMapEnvVar := k8score.EnvVar{
 				Name: keyToEnv.GetEnvVar(),
 				ValueFrom: &k8score.EnvVarSource{
-					ConfigMapKeyRef: &k8score.ConfigMapKeySelector{
-						Key: keyToEnv.GetConfigMapKey(),
-					},
+					ConfigMapKeyRef: configMapKeySelector,
 				},
 			}