Implement e2e TLS test.

Signed-off-by: agoins <[email protected]>

Author: alyssacgoins

.github/actions/kfp-cluster/action.yml

@@ -59,8 +59,8 @@ runs:
           ARGS="${ARGS} --cache-disabled"
         elif [ "${{inputs.pipeline_store }}" = "kubernetes" ]; then
           ARGS="${ARGS}  --deploy-k8s-native"
-        elif [ "${{inputs.pod_to_pod_tls_enabled }}" = "false"]; then
-          ARGS="${ARGS} --tls-enabled
+        elif [ "${{inputs.pod_to_pod_tls_enabled }}" = "true" ]; then
+          ARGS="${ARGS} --tls-enabled"
         fi
         
         ./.github/resources/scripts/deploy-kfp.sh $ARGS

.github/resources/manifests/argo/overlays/tls-enabled/apiserver-env.yaml

@@ -1,17 +1,18 @@
-apiVersion: v1
-kind: Pod
+apiVersion: apps/v1
+kind: Deployment
 metadata:
-  name: title
-  labels:
-    role: title
+  name: ml-pipeline
 spec:
-  containers:
-    - name: title
-      image: nginx
-      imagePullPolicy: IfNotPresent
-      ports:
-        - name: title
-          containerPort: 80
-          protocol: TCP
-  restartPolicy: Always
-  
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: V2_DRIVER_IMAGE
+              value: kind-registry:5000/driver
+            - name: V2_LAUNCHER_IMAGE
+              value: kind-registry:5000/launcher
+            - name: LOG_LEVEL
+              value: "debug"
+            - name: TLS_ENABLED
+              value: "true"

.github/resources/manifests/argo/overlays/tls-enabled/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../../../../../manifests/kustomize/env/cert-manager/platform-agnostic-multi-user-tls
+patches:
+  - path: apiserver-env.yaml
+    target:
+      kind: Deployment
+      name: ml-pipeline

.github/resources/scripts/deploy-kfp.sh

@@ -28,7 +28,7 @@ TEST_MANIFESTS=".github/resources/manifests/argo"
 PIPELINES_STORE="database"
 USE_PROXY=false
 CACHE_DISABLED=false
-POD_TO_POD_TLS_ENABLED=true
+POD_TO_POD_TLS_ENABLED=false
 
 # Loop over script arguments passed. This uses a single switch-case
 # block with default value in case we want to make alternative deployments
@@ -47,10 +47,14 @@ while [ "$#" -gt 0 ]; do
       CACHE_DISABLED=true
       shift
       ;;
+    --tls-enabled)
+      POD_TO_POD_TLS_ENABLED=true
+      shift
+      ;;
   esac
 done
 
-if [ "${USE_PROXY}" == "true" && "${PIPELINES_STORE}" == "kubernetes" ]; then
+if [ "${USE_PROXY}" == "true" ] && [ "${PIPELINES_STORE}" == "kubernetes" ]; then
   echo "ERROR: Kubernetes Pipeline store cannot be deployed with proxy support."
   exit 1
 fi
@@ -81,7 +85,7 @@ elif $USE_PROXY; then
   TEST_MANIFESTS="${TEST_MANIFESTS}/overlays/proxy"
 elif [ "${PIPELINES_STORE}" == "kubernetes" ]; then
   TEST_MANIFESTS="${TEST_MANIFESTS}/overlays/kubernetes-native"
-if $POD_TO_POD_TLS_ENABLED; then
+elif $POD_TO_POD_TLS_ENABLED; then
   TEST_MANIFESTS="${TEST_MANIFESTS}/overlays/tls-enabled"
 else
   TEST_MANIFESTS="${TEST_MANIFESTS}/overlays/no-proxy"

.github/workflows/e2e-test.yml

@@ -293,6 +293,63 @@ jobs:
           name: kfp-api-integration-tests-v2-with-proxy-artifacts-k8s-${{ matrix.k8s_version }}
           path: /tmp/tmp*/*
 
+  api-integration-tests-v2-with-tls-enabled:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k8s_version: [ "v1.31.0" ]
+    name: API integration tests v2 with TLS enabled - K8s ${{  matrix.k8s_version }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Create KFP cluster
+        id: create-kfp-cluster
+        uses: ./.github/actions/kfp-cluster
+        with:
+          k8s_version: ${{ matrix.k8s_version }}
+          pod_to_pod_tls_enabled: 'true'
+        continue-on-error: true
+
+      - name: Forward API port
+        id: forward-api-port
+        if: ${{ steps.create-kfp-cluster.outcome == 'success' }}
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        continue-on-error: true
+
+      - name: Forward MLMD port
+        id: forward-mlmd-port
+        if: ${{ steps.forward-api-port.outcome == 'success' }}
+        run: kubectl -n kubeflow port-forward svc/metadata-grpc-service 8080:8080 &
+        continue-on-error: true
+
+      - name: API integration tests v2
+        id: tests
+        if: ${{ steps.forward-mlmd-port.outcome == 'success' }}
+        working-directory: ./backend/test/v2/integration
+        run: go test -v ./... -namespace kubeflow -args -runIntegrationTests=true -tls_enabled=true
+        env:
+          PULL_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+
+      - name: Collect failed logs
+        if: ${{ steps.create-kfp-cluster.outcome != 'success' || steps.forward-api-port.outcome != 'success' || steps.tests.outcome != 'success' }}
+        run: |
+          ./.github/resources/scripts/collect-logs.sh --ns kubeflow --output /tmp/tmp_pod_log.txt
+          exit 1
+
+      - name: Collect test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: kfp-api-integration-tests-v2-with-tls-enabled-artifacts-k8s-${{ matrix.k8s_version }}
+          path: /tmp/tmp*/*
+
   api-integration-tests-v2-with-cache-disabled:
     runs-on: ubuntu-latest
     strategy:

backend/src/apiserver/config/proxy/config.go

@@ -24,6 +24,7 @@ const (
 	HttpsProxyEnv       = "HTTPS_PROXY"
 	NoProxyEnv          = "NO_PROXY"
 	defaultNoProxyValue = "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,metadata-grpc-service,0,1,2,3,4,5,6,7,8,9"
+	tlsEnabled          = "TLS_ENABLED"
 )
 
 type Config interface {

backend/src/v2/cacheutils/cache_test.go

@@ -135,7 +135,6 @@ func TestGenerateCacheKey(t *testing.T) {
 			wantErr: false,
 		},
 	}
-	//todo: add case fpr tlsEnabled testing? Also, should tlsEnabled be set to false here?
 	cacheClient, err := NewClient(false, false)
 	require.NoError(t, err)
 	for _, test := range tests {
@@ -258,8 +257,6 @@ func TestGenerateFingerPrint(t *testing.T) {
 			fingerPrint: "0a4cc1f15cdfad5170e1358518f7128c5278500a670db1b9a3f3d83b93db396e",
 		},
 	}
-	//todo: add case fpr tlsEnabled testing? Also, should tlsEnabled be set to false here?
-	//TODO: fixed
 	cacheClient, err := NewClient(false, false)
 	require.NoError(t, err)
 	for _, test := range tests {

backend/src/v2/client_manager/client_manager.go

@@ -26,9 +26,10 @@ type ClientManager struct {
 }
 
 type Options struct {
-	MLMDServerAddress string
-	MLMDServerPort    string
-	CacheDisabled     bool
+	MLMDServerAddress    string
+	MLMDServerPort       string
+	CacheDisabled        bool
+	MLPipelineTLSEnabled bool
 }
 
 // NewClientManager creates and Init a new instance of ClientManager.
@@ -63,7 +64,7 @@ func (cm *ClientManager) init(opts *Options) error {
 	if err != nil {
 		return err
 	}
-	cacheClient, err := initCacheClient(opts.CacheDisabled)
+	cacheClient, err := initCacheClient(opts.CacheDisabled, opts.MLPipelineTLSEnabled)
 	if err != nil {
 		return err
 	}
@@ -89,7 +90,6 @@ func initMetadataClient(address string, port string) (metadata.ClientInterface,
 	return metadata.NewClient(address, port)
 }
 
-// todo: should this be auto-set to false
-func initCacheClient(cacheDisabled bool) (cacheutils.Client, error) {
-	return cacheutils.NewClient(cacheDisabled, false)
+func initCacheClient(cacheDisabled bool, mlPipelineServiceTLSEnabled bool) (cacheutils.Client, error) {
+	return cacheutils.NewClient(cacheDisabled, mlPipelineServiceTLSEnabled)
 }

backend/src/v2/cmd/driver/main.go

@@ -87,7 +87,7 @@ var (
 	publishLogs       = flag.String("publish_logs", "true", "Whether to publish component logs to the object store")
 	cacheDisabledFlag = flag.Bool("cache_disabled", false, "Disable cache globally.")
 
-	mlPipelineServiceTLSEnabledStr = flag.String("mlPipelineServiceTLSEnabled", "false", "Set to 'true' if mlpipeline api server serves over TLS (default: 'false').")
+	mlPipelineServiceTLSEnabledStr = flag.String("ml_pipeline_service_tls_enabled", "false", "Set to 'true' if mlpipeline api server serves over TLS (default: 'false').")
 )
 
 // func RootDAG(pipelineName string, runID string, component *pipelinespec.ComponentSpec, task *pipelinespec.PipelineTaskSpec, mlmd *metadata.Client) (*Execution, error) {
@@ -191,20 +191,20 @@ func drive() (err error) {
 		return err
 	}
 	options := driver.Options{
-		PipelineName:     *pipelineName,
-		RunID:            *runID,
-		RunName:          *runName,
-		RunDisplayName:   *runDisplayName,
-		Namespace:        namespace,
-		Component:        componentSpec,
-		Task:             taskSpec,
-		DAGExecutionID:   *dagExecutionID,
-		IterationIndex:   *iterationIndex,
-		PipelineLogLevel: *logLevel,
-		PublishLogs:      *publishLogs,
-		CacheDisabled:    *cacheDisabledFlag,
-		DriverType:       *driverType,
-		TaskName:         *taskName,
+		PipelineName:         *pipelineName,
+		RunID:                *runID,
+		RunName:              *runName,
+		RunDisplayName:       *runDisplayName,
+		Namespace:            namespace,
+		Component:            componentSpec,
+		Task:                 taskSpec,
+		DAGExecutionID:       *dagExecutionID,
+		IterationIndex:       *iterationIndex,
+		PipelineLogLevel:     *logLevel,
+		PublishLogs:          *publishLogs,
+		CacheDisabled:        *cacheDisabledFlag,
+		DriverType:           *driverType,
+		TaskName:             *taskName,
 		MLPipelineTLSEnabled: mlPipelineServiceTLSEnabled,
 	}
 	var execution *driver.Execution

backend/src/v2/cmd/launcher-v2/main.go

@@ -46,7 +46,7 @@ var (
 	logLevel                       = flag.String("log_level", "1", "The verbosity level to log.")
 	publishLogs                    = flag.String("publish_logs", "true", "Whether to publish component logs to the object store")
 	cacheDisabledFlag              = flag.Bool("cache_disabled", false, "Disable cache globally.")
-	mlPipelineServiceTLSEnabledStr = flag.String("mlPipelineServiceTLSEnabled", "false", "Set to 'true' if mlpipeline api server serves over TLS (default: 'false').")
+	mlPipelineServiceTLSEnabledStr = flag.String("ml_pipeline_service_tls_enabled", "false", "Set to 'true' if mlpipeline api server serves over TLS (default: 'false').")
 )
 
 func main() {
@@ -111,9 +111,10 @@ func run() error {
 		return nil
 	case "container":
 		clientOptions := &client_manager.Options{
-			MLMDServerAddress: launcherV2Opts.MLMDServerAddress,
-			MLMDServerPort:    launcherV2Opts.MLMDServerPort,
-			CacheDisabled:     launcherV2Opts.CacheDisabled,
+			MLMDServerAddress:    launcherV2Opts.MLMDServerAddress,
+			MLMDServerPort:       launcherV2Opts.MLMDServerPort,
+			CacheDisabled:        launcherV2Opts.CacheDisabled,
+			MLPipelineTLSEnabled: launcherV2Opts.MLPipelineTLSEnabled,
 		}
 		clientManager, err := client_manager.NewClientManager(clientOptions)
 		if err != nil {