Device Plugin Args option

sriram-30 · sajmera-pensando · commit 044c6b479404 · 2025-03-26T18:49:17.000Z
diff --git a/api/v1alpha1/deviceconfig_types.go b/api/v1alpha1/deviceconfig_types.go
@@ -251,12 +251,11 @@ type DevicePluginSpec struct {
 	// +optional
 	DevicePluginTolerations []v1.Toleration `json:"devicePluginTolerations,omitempty"`
 
-	// resource naming strategy for device plugin
-	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ResourceNamingStrategy",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:ResourceNamingStrategy"}
-	// +kubebuilder:validation:Enum=single;mixed
-	// +kubebuilder:default:="single"
+	// device plugin arguments is used to pass supported flags and their values while starting device plugin daemonset
+	// supported flag values: {"resource_naming_strategy": {"single", "mixed"}}
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="DevicePluginArguments",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:devicePluginArguments"}
 	// +optional
-	ResourceNamingStrategy string `json:"resourceNamingStrategy,omitempty"`
+	DevicePluginArguments map[string]string `json:"devicePluginArguments,omitempty"`
 
 	// node labeller image
 	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="NodeLabellerImage",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:nodeLabellerImage"}
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml b/bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml
@@ -30,7 +30,7 @@ metadata:
         }
       ]
     capabilities: Basic Install
-    createdAt: "2025-03-21T05:09:41Z"
+    createdAt: "2025-03-25T06:19:27Z"
     operatorframework.io/suggested-namespace: openshift-amd-gpu
     operators.operatorframework.io/builder: operator-sdk-v1.32.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -152,6 +152,13 @@ spec:
         path: devicePlugin
         x-descriptors:
         - urn:alm:descriptor:com.amd.deviceconfigs:devicePlugin
+      - description: 'device plugin arguments is used to pass supported flags and
+          their values while starting device plugin daemonset supported flag values:
+          {"resource_naming_strategy": {"single", "mixed"}}'
+        displayName: DevicePluginArguments
+        path: devicePlugin.devicePluginArguments
+        x-descriptors:
+        - urn:alm:descriptor:com.amd.deviceconfigs:devicePluginArguments
       - description: device plugin image
         displayName: DevicePluginImage
         path: devicePlugin.devicePluginImage
@@ -192,11 +199,6 @@ spec:
         path: devicePlugin.nodeLabellerTolerations
         x-descriptors:
         - urn:alm:descriptor:com.amd.deviceconfigs:nodeLabellerTolerations
-      - description: resource naming strategy for device plugin
-        displayName: ResourceNamingStrategy
-        path: devicePlugin.resourceNamingStrategy
-        x-descriptors:
-        - urn:alm:descriptor:com.amd.deviceconfigs:ResourceNamingStrategy
       - description: upgrade policy for device plugin and node labeller daemons
         displayName: UpgradePolicy
         path: devicePlugin.upgradePolicy
diff --git a/bundle/manifests/amd.com_deviceconfigs.yaml b/bundle/manifests/amd.com_deviceconfigs.yaml
@@ -190,6 +190,13 @@ spec:
               devicePlugin:
                 description: device plugin
                 properties:
+                  devicePluginArguments:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      device plugin arguments is used to pass supported flags and their values while starting device plugin daemonset
+                      supported flag values: {"resource_naming_strategy": {"single", "mixed"}}
+                    type: object
                   devicePluginImage:
                     description: device plugin image
                     pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$
@@ -306,13 +313,6 @@ spec:
                           type: string
                       type: object
                     type: array
-                  resourceNamingStrategy:
-                    default: single
-                    description: resource naming strategy for device plugin
-                    enum:
-                    - single
-                    - mixed
-                    type: string
                   upgradePolicy:
                     description: upgrade policy for device plugin and node labeller
                       daemons
diff --git a/config/crd/bases/amd.com_deviceconfigs.yaml b/config/crd/bases/amd.com_deviceconfigs.yaml
@@ -186,6 +186,13 @@ spec:
               devicePlugin:
                 description: device plugin
                 properties:
+                  devicePluginArguments:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      device plugin arguments is used to pass supported flags and their values while starting device plugin daemonset
+                      supported flag values: {"resource_naming_strategy": {"single", "mixed"}}
+                    type: object
                   devicePluginImage:
                     description: device plugin image
                     pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$
@@ -302,13 +309,6 @@ spec:
                           type: string
                       type: object
                     type: array
-                  resourceNamingStrategy:
-                    default: single
-                    description: resource naming strategy for device plugin
-                    enum:
-                    - single
-                    - mixed
-                    type: string
                   upgradePolicy:
                     description: upgrade policy for device plugin and node labeller
                       daemons
diff --git a/config/manifests/bases/amd-gpu-operator.clusterserviceversion.yaml b/config/manifests/bases/amd-gpu-operator.clusterserviceversion.yaml
@@ -123,6 +123,13 @@ spec:
         path: devicePlugin
         x-descriptors:
         - urn:alm:descriptor:com.amd.deviceconfigs:devicePlugin
+      - description: 'device plugin arguments is used to pass supported flags and
+          their values while starting device plugin daemonset supported flag values:
+          {"resource_naming_strategy": {"single", "mixed"}}'
+        displayName: DevicePluginArguments
+        path: devicePlugin.devicePluginArguments
+        x-descriptors:
+        - urn:alm:descriptor:com.amd.deviceconfigs:devicePluginArguments
       - description: device plugin image
         displayName: DevicePluginImage
         path: devicePlugin.devicePluginImage
@@ -163,11 +170,6 @@ spec:
         path: devicePlugin.nodeLabellerTolerations
         x-descriptors:
         - urn:alm:descriptor:com.amd.deviceconfigs:nodeLabellerTolerations
-      - description: resource naming strategy for device plugin
-        displayName: ResourceNamingStrategy
-        path: devicePlugin.resourceNamingStrategy
-        x-descriptors:
-        - urn:alm:descriptor:com.amd.deviceconfigs:ResourceNamingStrategy
       - description: upgrade policy for device plugin and node labeller daemons
         displayName: UpgradePolicy
         path: devicePlugin.upgradePolicy
diff --git a/helm-charts-k8s/Chart.lock b/helm-charts-k8s/Chart.lock
@@ -6,4 +6,4 @@ dependencies:
   repository: file://./charts/kmm
   version: v1.0.0
 digest: sha256:f9a315dd2ce3d515ebf28c8e9a6a82158b493ca2686439ec381487761261b597
-generated: "2025-03-21T05:09:30.645342377Z"
+generated: "2025-03-25T06:19:17.248998622Z"
diff --git a/helm-charts-k8s/crds/deviceconfig-crd.yaml b/helm-charts-k8s/crds/deviceconfig-crd.yaml
@@ -194,6 +194,13 @@ spec:
               devicePlugin:
                 description: device plugin
                 properties:
+                  devicePluginArguments:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      device plugin arguments is used to pass supported flags and their values while starting device plugin daemonset
+                      supported flag values: {"resource_naming_strategy": {"single", "mixed"}}
+                    type: object
                   devicePluginImage:
                     description: device plugin image
                     pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$
@@ -310,13 +317,6 @@ spec:
                           type: string
                       type: object
                     type: array
-                  resourceNamingStrategy:
-                    default: single
-                    description: resource naming strategy for device plugin
-                    enum:
-                    - single
-                    - mixed
-                    type: string
                   upgradePolicy:
                     description: upgrade policy for device plugin and node labeller
                       daemons
diff --git a/helm-charts-openshift/Chart.lock b/helm-charts-openshift/Chart.lock
@@ -6,4 +6,4 @@ dependencies:
   repository: file://./charts/kmm
   version: v1.0.0
 digest: sha256:25200c34a5cc846a1275e5bf3fc637b19e909dc68de938189c5278d77d03f5ac
-generated: "2025-03-21T05:09:40.013067636Z"
+generated: "2025-03-25T06:19:26.060856628Z"
diff --git a/helm-charts-openshift/crds/deviceconfig-crd.yaml b/helm-charts-openshift/crds/deviceconfig-crd.yaml
@@ -194,6 +194,13 @@ spec:
               devicePlugin:
                 description: device plugin
                 properties:
+                  devicePluginArguments:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      device plugin arguments is used to pass supported flags and their values while starting device plugin daemonset
+                      supported flag values: {"resource_naming_strategy": {"single", "mixed"}}
+                    type: object
                   devicePluginImage:
                     description: device plugin image
                     pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[a-z0-9]+(?:[._-][a-z0-9]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$
@@ -310,13 +317,6 @@ spec:
                           type: string
                       type: object
                     type: array
-                  resourceNamingStrategy:
-                    default: single
-                    description: resource naming strategy for device plugin
-                    enum:
-                    - single
-                    - mixed
-                    type: string
                   upgradePolicy:
                     description: upgrade policy for device plugin and node labeller
                       daemons
diff --git a/internal/kmmmodule/kmmmodule.go b/internal/kmmmodule/kmmmodule.go
@@ -257,8 +257,14 @@ func (km *kmmModule) SetDevicePluginAsDesired(ds *appsv1.DaemonSet, devConfig *a
 		return fmt.Errorf("daemon set is not initialized, zero pointer")
 	}
 
-	resourceNamingStrategy := devConfig.Spec.DevicePlugin.ResourceNamingStrategy
-	command := []string{"sh", "-c", fmt.Sprintf("./k8s-device-plugin -logtostderr=true -stderrthreshold=INFO -v=5 -pulse=30 -resource_naming_strategy=%s", resourceNamingStrategy)}
+	commandArgs := "./k8s-device-plugin -logtostderr=true -stderrthreshold=INFO -v=5 -pulse=30"
+
+	devicePluginArguments := devConfig.Spec.DevicePlugin.DevicePluginArguments
+	for key, val := range devicePluginArguments {
+		commandArgs += " -" + key + "=" + val
+	}
+
+	command := []string{"sh", "-c", commandArgs}
 	nodeSelector := map[string]string{}
 	for key, val := range devConfig.Spec.Selector {
 		nodeSelector[key] = val
diff --git a/internal/utils.go b/internal/utils.go
@@ -32,10 +32,13 @@ import (
 )
 
 const (
-	defaultOcDriversVersion = "6.2.2"
-	openShiftNodeLabel      = "node.openshift.io/os_id"
-	NodeFeatureLabelAmdGpu  = "feature.node.kubernetes.io/amd-gpu"
-	NodeFeatureLabelAmdVGpu = "feature.node.kubernetes.io/amd-vgpu"
+	defaultOcDriversVersion    = "6.2.2"
+	openShiftNodeLabel         = "node.openshift.io/os_id"
+	NodeFeatureLabelAmdGpu     = "feature.node.kubernetes.io/amd-gpu"
+	NodeFeatureLabelAmdVGpu    = "feature.node.kubernetes.io/amd-vgpu"
+	ResourceNamingStrategyFlag = "resource_naming_strategy"
+	SingleStrategy             = "single"
+	MixedStrategy              = "mixed"
 )
 
 func GetDriverVersion(node v1.Node, deviceConfig amdv1alpha1.DeviceConfig) (string, error) {
diff --git a/internal/validator/specValidators.go b/internal/validator/specValidators.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	amdv1alpha1 "github.com/ROCm/gpu-operator/api/v1alpha1"
+	utils "github.com/ROCm/gpu-operator/internal"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -86,5 +87,29 @@ func ValidateDevicePluginSpec(ctx context.Context, client client.Client, devConf
 		}
 	}
 
+	supportedFlagValues := map[string][]string{
+		utils.ResourceNamingStrategyFlag: {utils.SingleStrategy, utils.MixedStrategy},
+	}
+
+	devicePluginArguments := devConfig.Spec.DevicePlugin.DevicePluginArguments
+	for key, val := range devicePluginArguments {
+		validValues, validKey := supportedFlagValues[key]
+		if !validKey {
+			return fmt.Errorf("Invalid flag: %s", key)
+		}
+		validKeyValue := false
+
+		for _, validVal := range validValues {
+			if val == validVal {
+				validKeyValue = true
+				break
+			}
+		}
+
+		if !validKeyValue {
+			return fmt.Errorf("Invalid flag value: %s=%s. Supported values: %v", key, val, supportedFlagValues[key])
+		}
+	}
+
 	return nil
 }
diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile
@@ -7,17 +7,23 @@ E2E_KUBE_RBAC_PROXY_CURL_IMAGE ?= curlimages/curl:7.78.0
 E2E_UBUNTU_BASE_IMAGE ?= ubuntu:22.04
 E2E_MINIO_IMAGE ?= minio/minio:latest
 E2E_EXPORTER_IMAGE ?= rocm/device-metrics-exporter:v1.2.0
+E2E_EXPORTER_IMAGE_2 ?= rocm/device-metrics-exporter:v1.1.1-beta.0
 E2E_DEVICE_PLUGIN_IMAGE ?= rocm/k8s-device-plugin:latest
 E2E_NODE_LABELLER_IMAGE ?= rocm/k8s-device-plugin:labeller-latest
+E2E_DEVICE_PLUGIN_IMAGE_2 ?= rocm/k8s-device-plugin:1.31.0.6
+E2E_NODE_LABELLER_IMAGE_2 ?= rocm/k8s-device-plugin:labeller-1.31.0.6
 E2E_TEST_RUNNER_IMAGE ?= rocm/test-runner:v1.2.0-beta.0
 
 export E2E_INIT_CONTAINER_IMAGE
 export E2E_KUBE_RBAC_PROXY_CURL_IMAGE
 export E2E_UBUNTU_BASE_IMAGE
 export E2E_MINIO_IMAGE
 export E2E_EXPORTER_IMAGE
+export E2E_EXPORTER_IMAGE_2
 export E2E_DEVICE_PLUGIN_IMAGE
 export E2E_NODE_LABELLER_IMAGE
+export E2E_DEVICE_PLUGIN_IMAGE_2
+export E2E_NODE_LABELLER_IMAGE_2
 export E2E_TEST_RUNNER_IMAGE
 
 export E2E_DCM_IMAGE
diff --git a/tests/e2e/cluster_test.go b/tests/e2e/cluster_test.go
@@ -1877,8 +1877,8 @@ func (s *E2ESuite) TestDevicePluginNodeLabellerDaemonSetUpgrade(c *C) {
 
 	// upgrade
 	// update the CR's device plugin with image
-	devCfg.Spec.DevicePlugin.DevicePluginImage = devicePluginImage
-	devCfg.Spec.DevicePlugin.NodeLabellerImage = nodeLabellerImage
+	devCfg.Spec.DevicePlugin.DevicePluginImage = devicePluginImage2
+	devCfg.Spec.DevicePlugin.NodeLabellerImage = nodeLabellerImage2
 	s.patchDevicePluginImage(devCfg, c)
 	s.patchNodeLabellerImage(devCfg, c)
 	s.verifyDevicePluginStatus(s.ns, c, devCfg)
@@ -1911,7 +1911,7 @@ func (s *E2ESuite) TestMetricsExporterDaemonSetUpgrade(c *C) {
 
 	// upgrade
 	// update the CR's device plugin with image
-	devCfg.Spec.MetricsExporter.Image = exporterImage
+	devCfg.Spec.MetricsExporter.Image = exporterImage2
 	s.patchMetricsExporterImage(devCfg, c)
 	s.verifyDeviceConfigStatus(devCfg, c)
 	s.checkMetricsExporterStatus(devCfg, s.ns, v1.ServiceTypeClusterIP, c)
diff --git a/tests/e2e/init.go b/tests/e2e/init.go
@@ -25,8 +25,11 @@ var (
 	initContainerImage     string
 	kubeRbacProxyCurlImage string
 	exporterImage          string
+	exporterImage2         string
 	devicePluginImage      string
 	nodeLabellerImage      string
+	devicePluginImage2     string
+	nodeLabellerImage2     string
 	testRunnerImage        string
 	driverImageRepo        string
 )
@@ -46,6 +49,10 @@ func init() {
 	if !ok {
 		log.Fatalf("E2E_EXPORTER_IMAGE is not defined")
 	}
+	exporterImage2, ok = os.LookupEnv("E2E_EXPORTER_IMAGE_2")
+	if !ok {
+		log.Fatalf("E2E_EXPORTER_IMAGE_2 is not defined")
+	}
 	devicePluginImage, ok = os.LookupEnv("E2E_DEVICE_PLUGIN_IMAGE")
 	if !ok {
 		log.Fatalf("E2E_DEVICE_PLUGIN_IMAGE is not defined")
@@ -54,6 +61,14 @@ func init() {
 	if !ok {
 		log.Fatalf("E2E_NODE_LABELLER_IMAGE is not defined")
 	}
+	devicePluginImage2, ok = os.LookupEnv("E2E_DEVICE_PLUGIN_IMAGE_2")
+	if !ok {
+		log.Fatalf("E2E_DEVICE_PLUGIN_IMAGE_2 is not defined")
+	}
+	nodeLabellerImage2, ok = os.LookupEnv("E2E_NODE_LABELLER_IMAGE_2")
+	if !ok {
+		log.Fatalf("E2E_NODE_LABELLER_IMAGE_2 is not defined")
+	}
 	testRunnerImage, ok = os.LookupEnv("E2E_TEST_RUNNER_IMAGE")
 	if !ok {
 		log.Fatalf("E2E_TEST_RUNNER_IMAGE is not defined")
