add e2e test to verify workspace connection

Signed-off-by: Harshad Reddy Nalla <[email protected]>

Author: harshad16

workspaces/controller/test/e2e/e2e_suite_test.go

@@ -120,6 +120,11 @@ var _ = BeforeSuite(func() {
 			Expect(utils.InstallIstioIngressGateway(istioNamespace)).To(Succeed(), "Failed to install istio ingress gateway")
 		} else {
 			_, _ = fmt.Fprintf(GinkgoWriter, "WARNING: istio ingress gateway is already installed. Skipping installation...\n")
+			// Check if Gateway CR exists, if not create
+			if !utils.IsIngressGatewayCRInstalled(istioNamespace) {
+				_, _ = fmt.Fprintf(GinkgoWriter, "Gateway CR not found, creating it...\n")
+				Expect(utils.CreateIngressGatewayCR(istioNamespace)).To(Succeed(), "Failed to create Gateway CR")
+			}
 		}
 		By("checking that istio ingress gateway is available")
 		Expect(utils.WaitIstioIngressGatewayReady(istioNamespace)).To(Succeed(), "istio ingress gateway is not available")

workspaces/controller/test/e2e/e2e_test.go

@@ -77,19 +77,19 @@ var _ = Describe("controller", Ordered, func() {
 		_, _ = utils.Run(cmd) // ignore errors because namespace may already exist
 
 		// TODO: enable Istio injection once we have logic to create VirtualServices during Workspace reconciliation
-		// By("labeling namespaces for Istio injection")
-		// err := utils.LabelNamespaceForIstioInjection(controllerNamespace)
-		// ExpectWithOffset(1, err).NotTo(HaveOccurred())
+		By("labeling namespaces for Istio injection")
+		err := utils.LabelNamespaceForIstioInjection(controllerNamespace)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
-		// err = utils.LabelNamespaceForIstioInjection(workspaceNamespace)
-		// ExpectWithOffset(1, err).NotTo(HaveOccurred())
+		err = utils.LabelNamespaceForIstioInjection(workspaceNamespace)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 		By("creating common workspace resources")
 		cmd = exec.Command("kubectl", "apply",
 			"-k", filepath.Join(projectDir, "config/samples/common"),
 			"-n", workspaceNamespace,
 		)
-		_, err := utils.Run(cmd)
+		_, err = utils.Run(cmd)
 		ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 		By("installing CRDs")
@@ -98,6 +98,11 @@ var _ = Describe("controller", Ordered, func() {
 		ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 		By("deploying the workspaces-controller")
+		// Update the ISTIO_GATEWAY config to use istio-system/istio-ingressgateway
+		cmd = exec.Command("sed", "-i", "s|ISTIO_GATEWAY=.*|ISTIO_GATEWAY=istio-system/istio-ingressgateway|", "config/components/istio/params.env")
+		_, err = utils.Run(cmd)
+		ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
 		cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", controllerImage))
 		_, err = utils.Run(cmd)
 		ExpectWithOffset(1, err).NotTo(HaveOccurred())
@@ -298,7 +303,7 @@ var _ = Describe("controller", Ordered, func() {
 			curlService := func() error {
 				// NOTE: this command should exit with a non-zero status code if the HTTP status code is >= 400
 				cmd := exec.Command("kubectl", "run",
-					"tmp-curl", "-n", workspaceNamespace,
+					"tmp-curl", "-n", workspaceNamespace, "--labels", "sidecar.istio.io/inject=false",
 					"--attach", "--command", fmt.Sprintf("--image=%s", curlImage), "--rm", "--restart=Never", "--",
 					"curl", "-sSL", "-o", "/dev/null", "--fail-with-body", serviceEndpoint,
 				)
@@ -307,6 +312,60 @@ var _ = Describe("controller", Ordered, func() {
 			}
 			Eventually(curlService, timeout, interval).Should(Succeed())
 
+			By("validating that the workspace virtual service was created")
+			var workspaceVirtualServiceName string
+			verifyWorkspaceVirtualService := func(g Gomega) {
+				cmd := exec.Command("kubectl", "get", "virtualservices",
+					"-l", fmt.Sprintf("notebooks.kubeflow.org/workspace-name=%s", workspaceName),
+					"-n", workspaceNamespace,
+					"-o", "go-template={{ range .items }}"+
+						"{{ if not .metadata.deletionTimestamp }}"+
+						"{{ .metadata.name }}"+
+						"{{ \"\\n\" }}{{ end }}{{ end }}",
+				)
+				vsOutput, err := utils.Run(cmd)
+				g.Expect(err).NotTo(HaveOccurred())
+
+				// Ensure only 1 virtual service is found
+				vsNames := utils.GetNonEmptyLines(vsOutput)
+				g.Expect(vsNames).To(HaveLen(1), "expected 1 virtual service found")
+				workspaceVirtualServiceName = vsNames[0]
+				g.Expect(workspaceVirtualServiceName).To(ContainSubstring(fmt.Sprintf("ws-%s", workspaceName)))
+			}
+			Eventually(verifyWorkspaceVirtualService, timeout, interval).Should(Succeed())
+
+			By("validating that the workspace virtual service endpoint is reachable via Istio gateway")
+			// Start port-forward to istio-ingressgateway service
+			// The service exposes HTTP on port 80
+			localPort := "18080"
+			serviceHTTPPort := "80"
+			portForwardSpec := fmt.Sprintf("%s:%s", localPort, serviceHTTPPort)
+
+			pf, err := utils.StartPortForward(istioNamespace, "istio-ingressgateway", portForwardSpec, 30*time.Second)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			defer pf.Stop()
+
+			// Give port-forward a moment to stabilize
+			time.Sleep(2 * time.Second)
+
+			// Test the workspace endpoint through the Istio gateway
+			gatewayEndpoint := fmt.Sprintf("http://localhost:%s/workspace/connect/%s/%s/%s/lab",
+				localPort, workspaceNamespace, workspaceName, workspacePortId,
+			)
+			_, _ = fmt.Fprintf(GinkgoWriter, "Testing gateway endpoint: %s\n", gatewayEndpoint)
+
+			testGatewayEndpoint := func() error {
+				cmd := exec.Command("curl", "-sSL", "-o", "/dev/null", "--fail-with-body", "-w", "%{http_code}", gatewayEndpoint)
+				output, err := utils.Run(cmd)
+				if err != nil {
+					return fmt.Errorf("curl failed: %w (HTTP status: %s)", err, output)
+				}
+				_, _ = fmt.Fprintf(GinkgoWriter, "Gateway endpoint returned HTTP status: %s\n", output)
+				return nil
+			}
+			Eventually(testGatewayEndpoint, timeout, interval).Should(Succeed(),
+				"Workspace should be reachable through Istio gateway at %s", gatewayEndpoint)
+
 			By("ensuring in-use imageConfig values cannot be removed from WorkspaceKind")
 			removeInUseImageConfig := func() error {
 				cmd := exec.Command("kubectl", "patch", "workspacekind", workspaceKindName,

workspaces/controller/test/utils/common.go

@@ -17,10 +17,13 @@ limitations under the License.
 package utils
 
 import (
+	"bufio"
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/onsi/ginkgo/v2"
 )
@@ -72,3 +75,90 @@ func GetProjectDir() (string, error) {
 	wd = strings.ReplaceAll(wd, "/test/e2e", "")
 	return wd, nil
 }
+
+// PortForward represents a running port-forward session
+type PortForward struct {
+	cmd    *exec.Cmd
+	cancel context.CancelFunc
+}
+
+// Stop stops the port-forward session
+func (pf *PortForward) Stop() {
+	if pf.cancel != nil {
+		pf.cancel()
+	}
+	if pf.cmd != nil && pf.cmd.Process != nil {
+		_ = pf.cmd.Process.Kill()
+	}
+}
+
+// StartPortForward starts a port-forward to a service in the background.
+// Returns a PortForward handle that can be used to stop the port-forward.
+// The port-forward is considered ready when it starts listening (output contains "Forwarding from").
+func StartPortForward(namespace, service, ports string, readyTimeout time.Duration) (*PortForward, error) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	cmd := exec.CommandContext(ctx, "kubectl", "port-forward",
+		"-n", namespace,
+		"service/"+service,
+		ports,
+	)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		cancel()
+		return nil, fmt.Errorf("failed to get stdout pipe: %w", err)
+	}
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		cancel()
+		return nil, fmt.Errorf("failed to get stderr pipe: %w", err)
+	}
+
+	if err := cmd.Start(); err != nil {
+		cancel()
+		return nil, fmt.Errorf("failed to start port-forward: %w", err)
+	}
+
+	pf := &PortForward{cmd: cmd, cancel: cancel}
+
+	// Monitor stdout for ready signal
+	readyChan := make(chan error, 1)
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		for scanner.Scan() {
+			line := scanner.Text()
+			_, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "port-forward stdout: %s\n", line)
+			if strings.Contains(line, "Forwarding from") {
+				readyChan <- nil
+				return
+			}
+		}
+		if err := scanner.Err(); err != nil {
+			readyChan <- fmt.Errorf("error reading stdout: %w", err)
+		}
+	}()
+
+	// Log stderr in background
+	go func() {
+		scanner := bufio.NewScanner(stderr)
+		for scanner.Scan() {
+			_, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "port-forward stderr: %s\n", scanner.Text())
+		}
+	}()
+
+	// Wait for ready signal or timeout
+	select {
+	case err := <-readyChan:
+		if err != nil {
+			pf.Stop()
+			return nil, fmt.Errorf("port-forward failed to become ready: %w", err)
+		}
+		_, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "port-forward is ready\n")
+		return pf, nil
+	case <-time.After(readyTimeout):
+		pf.Stop()
+		return nil, fmt.Errorf("port-forward did not become ready within %v", readyTimeout)
+	}
+}

workspaces/controller/test/utils/istio.go

@@ -74,9 +74,15 @@ func buildIstioIngressGatewayParams(command string, istioNamespace string) []str
 }
 
 func UninstallIstioIngressGateway(istioNamespace string) {
+	// Delete the Gateway CR first
+	cmd := exec.Command("kubectl", "delete", "gateway", istioIngressGatewayName, "-n", istioNamespace, "--ignore-not-found=true")
+	if _, err := Run(cmd); err != nil {
+		warnError(fmt.Errorf("failed to delete Gateway CR: %w", err))
+	}
+
 	// Uninstall Istio ingress gateway using the same base params as installation
 	params := buildIstioIngressGatewayParams("uninstall", istioNamespace)
-	cmd := exec.Command(getIstioctlPath(), params...)
+	cmd = exec.Command(getIstioctlPath(), params...)
 	if _, err := Run(cmd); err != nil {
 		warnError(fmt.Errorf("failed to uninstall Istio ingress gateway: %w", err))
 		return
@@ -119,6 +125,44 @@ func InstallIstioIngressGateway(istioNamespace string) error {
 	}
 
 	fmt.Println("Istio ingress gateway installation completed")
+
+	// Create the Gateway CR in the same namespace as the ingress gateway
+	fmt.Printf("Creating Gateway CR in namespace %s...\n", istioNamespace)
+	if err := CreateIngressGatewayCR(istioNamespace); err != nil {
+		return fmt.Errorf("failed to create Gateway CR: %w", err)
+	}
+
+	return nil
+}
+
+// CreateIngressGatewayCR creates a Gateway CR for the istio-ingressgateway
+func CreateIngressGatewayCR(istioNamespace string) error {
+	// Create the Gateway CR in the istio namespace
+	// This configures the istio-ingressgateway to accept HTTP/HTTPS traffic
+	gatewayYAML := fmt.Sprintf(`apiVersion: networking.istio.io/v1beta1
+kind: Gateway
+metadata:
+  name: %s
+  namespace: %s
+spec:
+  selector:
+    istio: ingressgateway
+  servers:
+  - port:
+      number: 80
+      name: http
+      protocol: HTTP
+    hosts:
+    - "*"
+`, istioIngressGatewayName, istioNamespace)
+
+	cmd := exec.Command("kubectl", "apply", "-f", "-")
+	cmd.Stdin = strings.NewReader(gatewayYAML)
+	if _, err := Run(cmd); err != nil {
+		return fmt.Errorf("failed to create Gateway CR: %w", err)
+	}
+
+	fmt.Printf("Gateway CR '%s' created in namespace %s\n", istioIngressGatewayName, istioNamespace)
 	return nil
 }
 
@@ -266,3 +310,13 @@ func LabelNamespaceForIstioInjection(namespace string) error {
 	}
 	return nil
 }
+
+// IsIngressGatewayCRInstalled checks if the Gateway CR exists in the istio namespace
+func IsIngressGatewayCRInstalled(istioNamespace string) bool {
+	cmd := exec.Command("kubectl", "get", "gateway", istioIngressGatewayName, "-n", istioNamespace, "--ignore-not-found")
+	output, err := Run(cmd)
+	if err != nil {
+		return false
+	}
+	return strings.TrimSpace(output) != ""
+}