[Feature] GPUOP-309 Make in-cluster driver image build base image configurable (#766) (#234)

Author: yansun1996

api/v1alpha1/deviceconfig_types.go

@@ -140,6 +140,11 @@ type DriverSpec struct {
 	// +optional
 	ImageSign ImageSignSpec `json:"imageSign,omitempty"`
 
+	// image build configs
+	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ImageBuild",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:imageBuild"}
+	// +optional
+	ImageBuild ImageBuildSpec `json:"imageBuild,omitempty"`
+
 	// policy to upgrade the drivers
 	//+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="UpgradePolicy",xDescriptors={"urn:alm:descriptor:com.amd.deviceconfigs:upgradePolicy"}
 	// +optional
@@ -337,6 +342,17 @@ type ImageSignSpec struct {
 	CertSecret *v1.LocalObjectReference `json:"certSecret,omitempty"`
 }
 
+type ImageBuildSpec struct {
+	// image registry to fetch base image for building driver image, default value is docker.io, the builder will search for corresponding OS base image from given registry
+	// e.g. if your worker node is using Ubuntu 22.04, by default the base image would be docker.io/ubuntu:22.04
+	// NOTE: this field won't apply for OpenShift since OpenShift is using its own DriverToolKit image to build driver image
+	// +kubebuilder:default=docker.io
+	BaseImageRegistry string `json:"baseImageRegistry,omitempty"`
+
+	// TLS settings for fetching base image
+	BaseImageRegistryTLS RegistryTLS `json:"baseImageRegistryTLS,omitempty"`
+}
+
 // ServiceType string describes ingress methods for a service
 type ServiceType string
 

api/v1alpha1/zz_generated.deepcopy.go

@@ -32,7 +32,7 @@ metadata:
     capabilities: Seamless Upgrades
     categories: AI/Machine Learning,Monitoring
     containerImage: docker.io/rocm/gpu-operator:v1.2.0
-    createdAt: "2025-06-11T22:50:16Z"
+    createdAt: "2025-06-12T00:51:00Z"
     description: |-
       Operator responsible for deploying AMD GPU kernel drivers, device plugin, device test runner and device metrics exporter
       For more information, visit [documentation](https://instinct.docs.amd.com/projects/gpu-operator/en/latest/)
@@ -285,6 +285,22 @@ spec:
         path: driver.image
         x-descriptors:
         - urn:alm:descriptor:com.amd.deviceconfigs:image
+      - description: image build configs
+        displayName: ImageBuild
+        path: driver.imageBuild
+        x-descriptors:
+        - urn:alm:descriptor:com.amd.deviceconfigs:imageBuild
+      - description: If true, check if the container image already exists using plain
+          HTTP.
+        displayName: Insecure
+        path: driver.imageBuild.baseImageRegistryTLS.insecure
+        x-descriptors:
+        - urn:alm:descriptor:com.amd.deviceconfigs:insecure
+      - description: If true, skip any TLS server certificate validation
+        displayName: InsecureSkipTLSVerify
+        path: driver.imageBuild.baseImageRegistryTLS.insecureSkipTLSVerify
+        x-descriptors:
+        - urn:alm:descriptor:com.amd.deviceconfigs:insecureSkipTLSVerify
       - description: secrets used for pull/push images from/to private registry specified
           in driversImage
         displayName: ImageRegistrySecret

bundle/manifests/amd-gpu-operator.clusterserviceversion.yaml

@@ -384,6 +384,29 @@ spec:
                       NOTE: Updating the driver image repository is not supported. Please delete the existing DeviceConfig and create a new one with the updated image repository
                     pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[$a-zA-Z0-9_]+(?:[._-][$a-zA-Z0-9_]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$
                     type: string
+                  imageBuild:
+                    description: image build configs
+                    properties:
+                      baseImageRegistry:
+                        default: docker.io
+                        description: |-
+                          image registry to fetch base image for building driver image, default value is docker.io, the builder will search for corresponding OS base image from given registry
+                          e.g. if your worker node is using Ubuntu 22.04, by default the base image would be docker.io/ubuntu:22.04
+                          NOTE: this field won't apply for OpenShift since OpenShift is using its own DriverToolKit image to build driver image
+                        type: string
+                      baseImageRegistryTLS:
+                        description: TLS settings for fetching base image
+                        properties:
+                          insecure:
+                            description: If true, check if the container image already
+                              exists using plain HTTP.
+                            type: boolean
+                          insecureSkipTLSVerify:
+                            description: If true, skip any TLS server certificate
+                              validation
+                            type: boolean
+                        type: object
+                    type: object
                   imageRegistrySecret:
                     description: secrets used for pull/push images from/to private
                       registry specified in driversImage

bundle/manifests/amd.com_deviceconfigs.yaml

@@ -380,6 +380,29 @@ spec:
                       NOTE: Updating the driver image repository is not supported. Please delete the existing DeviceConfig and create a new one with the updated image repository
                     pattern: ^([a-z0-9]+(?:[._-][a-z0-9]+)*(:[0-9]+)?)(/[$a-zA-Z0-9_]+(?:[._-][$a-zA-Z0-9_]+)*)*(?::[a-z0-9._-]+)?(?:@[a-zA-Z0-9]+:[a-f0-9]+)?$
                     type: string
+                  imageBuild:
+                    description: image build configs
+                    properties:
+                      baseImageRegistry:
+                        default: docker.io
+                        description: |-
+                          image registry to fetch base image for building driver image, default value is docker.io, the builder will search for corresponding OS base image from given registry
+                          e.g. if your worker node is using Ubuntu 22.04, by default the base image would be docker.io/ubuntu:22.04
+                          NOTE: this field won't apply for OpenShift since OpenShift is using its own DriverToolKit image to build driver image
+                        type: string
+                      baseImageRegistryTLS:
+                        description: TLS settings for fetching base image
+                        properties:
+                          insecure:
+                            description: If true, check if the container image already
+                              exists using plain HTTP.
+                            type: boolean
+                          insecureSkipTLSVerify:
+                            description: If true, skip any TLS server certificate
+                              validation
+                            type: boolean
+                        type: object
+                    type: object
                   imageRegistrySecret:
                     description: secrets used for pull/push images from/to private
                       registry specified in driversImage

config/crd/bases/amd.com_deviceconfigs.yaml

@@ -256,6 +256,22 @@ spec:
         path: driver.image
         x-descriptors:
         - urn:alm:descriptor:com.amd.deviceconfigs:image
+      - description: image build configs
+        displayName: ImageBuild
+        path: driver.imageBuild
+        x-descriptors:
+        - urn:alm:descriptor:com.amd.deviceconfigs:imageBuild
+      - description: If true, check if the container image already exists using plain
+          HTTP.
+        displayName: Insecure
+        path: driver.imageBuild.baseImageRegistryTLS.insecure
+        x-descriptors:
+        - urn:alm:descriptor:com.amd.deviceconfigs:insecure
+      - description: If true, skip any TLS server certificate validation
+        displayName: InsecureSkipTLSVerify
+        path: driver.imageBuild.baseImageRegistryTLS.insecureSkipTLSVerify
+        x-descriptors:
+        - urn:alm:descriptor:com.amd.deviceconfigs:insecureSkipTLSVerify
       - description: secrets used for pull/push images from/to private registry specified
           in driversImage
         displayName: ImageRegistrySecret

config/manifests/bases/amd-gpu-operator.clusterserviceversion.yaml

@@ -4,7 +4,7 @@ The Device Config Manager (DCM) job is to monitor for and apply different config
 
 ## ConfigMap
 
-As mentioned, the `config.json` data specifies different GPU partitioning profiles that can be set on the GPU nodes in your cluster. Below is an example Device Config Manager ConfigMap. This example ConfigMap is also available in the GPU Operator repo here: [_example/configmap.yaml_](https://github.com/pensando/gpu-operator/blob/main/example/configManager/configmap.yaml)
+As mentioned, the `config.json` data specifies different GPU partitioning profiles that can be set on the GPU nodes in your cluster. Below is an example Device Config Manager ConfigMap. This example ConfigMap is also available in the GPU Operator repo here: [_example/configmap.yaml_](https://github.com/ROCm/gpu-operator/blob/main/example/configManager/configmap.yaml)
 
 ```yaml  
 apiVersion: v1

docs/dcm/device-config-manager-configmap.md

@@ -43,6 +43,7 @@ Below is an example of a full DeviceConfig CR that can be used to install the AM
         # Not working for OpenShift cluster. OpenShift users please use the Machine Config Operator (MCO) resource to configure amdgpu blacklist.
         # Example MCO resource is available at https://instinct.docs.amd.com/projects/gpu-operator/en/latest/installation/openshift-olm.html#create-blacklist-for-installing-out-of-tree-kernel-module
         blacklist: false
+        version: "6.4" # Specify the driver version you would like to be installed that coincides with a ROCm version number
         # Specify your repository to host driver image
         # Note:
         # 1. DO NOT include the image tag as AMD GPU Operator will automatically manage the image tag for you
@@ -53,14 +54,36 @@ Below is an example of a full DeviceConfig CR that can be used to install the AM
         # kubectl create secret docker-registry mysecret -n kmm-namespace --docker-username=xxx --docker-password=xxx
         # Make sure you created the secret within the namespace that KMM operator is running
         imageRegistrySecret:
-          name: mysecret
-        imageRegistryTLS:
-          insecure: false # If true, check for the container image using plain HTTP
-          insecureSkipTLSVerify: false # If true, skip any TLS server certificate validation (useful for self-signed certificates)
-      version: "6.3" # Specify the driver version you would like to be installed that coincides with a ROCm version number
-      upgradePolicy:
-        enable: true
-        maxParallelUpgrades: 3 # (Optional) Number of nodes that will be upgraded in parallel. Default is 1
+          name: my-image-secret
+        imageRegistryTLS: 
+          insecure: False # If True, check for the container image using plain HTTP
+          insecureSkipTLSVerify: False # If True, skip any TLS server certificate validation (useful for self-signed certificates)
+        upgradePolicy:
+          enable: true # (Optional) set to true to enable auto driver upgrade, set to false to manage driver upgrade manually
+          maxParallelUpgrades: 3 # (Optional) Number of nodes that will be upgraded in parallel. Default is 1
+        # (Optional) specify the secret that saves the private and public keys used to sign the built driver
+        # secure boot enabled node requires image signing to load the kernel module
+        # you need to register the public key in the system's Machine Owner Key (MOK) database
+        imageSign:
+          keySecret:
+            name: image-sign-private-key-secret
+          certSecret:
+            name: image-sign-public-key-secret
+        # (Optional) configure the driver image build within the cluster
+        imageBuild:
+          # configure the registry to search for base image for building driver
+          # e.g. if you are using worker node with ubuntu 22.04 and baseImageRegistry is docker.io
+          # image builder will use docker.io/ubuntu:22.04 as base image
+          baseImageRegistry: docker.io
+          baseImageRegistryTLS:
+            insecure: False # If True, check for the container image using plain HTTP
+            insecureSkipTLSVerify: False # If True, skip any TLS server certificate validation (useful for self-signed certificates)
+        # (Optional) specify driver toleration so operator can manage out-of-tree drivers on tainted nodes
+        tolerations:
+          - key: "example-key"
+            operator: "Equal"
+            value: "example-value"
+            effect: "NoSchedule"
       ## AMD K8s Device Plugin Configuration ##
       commonConfig:
         # (Optional) Specify common values used by all components. 

docs/fulldeviceconfig.rst

@@ -54,6 +54,10 @@ deviceConfig:
       version: "6.4"
       # -- specify the secrets to sign the out-of-tree kernel module inside driver image for secure boot, e.g. input private / public key secret {"keySecret":{"name":"privateKeySecret"},"certSecret":{"name":"publicKeySecret"}}
       imageSign: {}
+      # -- configure the out-of-tree driver image build within the cluster. e.g. {"baseImageRegistry":"docker.io","baseImageRegistryTLS":{"baseImageRegistry":"docker.io","baseImageRegistryTLS":{"insecure":"false","insecureSkipTLSVerify":"false"}}}
+      imageBuild: {}
+      # -- configure driver tolerations so that operator can manage out-of-tree drivers on tainted nodes
+      tolerations: []
       upgradePolicy:
         # -- enable/disable automatic driver upgrade feature 
         enable: true

hack/k8s-patch/metadata-patch/values.yaml

@@ -48,6 +48,16 @@ spec:
       {{- toYaml . | nindent 6 }}
     {{- end }}
 
+    {{- with .imageBuild }}
+    imageBuild:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+
+    {{- with .tolerations }}
+    tolerations:
+      {{- toYaml . | nindent 6 }}
+    {{- end }}
+
     {{- with .upgradePolicy }}
     upgradePolicy:
       {{- toYaml . | nindent 6 }}