feat: add istio virtualservice reconciliation to controller

workspaces/backend/internal/models/workspaces/funcs.go

@@ -359,7 +359,7 @@ func buildServices(ws *kubefloworgv1beta1.Workspace, wskPodTemplatePorts map[kub
 		case kubefloworgv1beta1.ImagePortProtocolHTTP:
 			services[i].HttpService = &HttpService{
 				DisplayName: ptr.Deref(port.DisplayName, wskPort.DefaultDisplayName),
-				HttpPath:    fmt.Sprintf("/workspace/%s/%s/%s/", ws.Namespace, ws.Name, port.Id),
+				HttpPath:    fmt.Sprintf("/workspace/connect/%s/%s/%s/", ws.Namespace, ws.Name, port.Id),
 			}
 		}
 	}

workspaces/controller/api/v1beta1/workspacekind_types.go

@@ -279,7 +279,7 @@ type WorkspaceKindVolumeMounts struct {
 
 type HTTPProxy struct {
 	// if the path prefix is stripped from incoming HTTP requests
-	//  - if true, the '/workspace/{profile_name}/{workspace_name}/' path prefix
+	//  - if true, the '/workspace/connect/{profile_name}/{workspace_name}/' path prefix
 	//    is stripped from incoming requests, the application sees the request
 	//    as if it was made to '/...'
 	//  - this only works if the application serves RELATIVE URLs for its assets
@@ -291,7 +291,7 @@ type HTTPProxy struct {
 	//  - sets the `spec.http[].headers.request` of the Istio VirtualService
 	//    https://istio.io/latest/docs/reference/config/networking/virtual-service/#Headers-HeaderOperations
 	//  - the following string templates are available:
-	//     - `.PathPrefix`: the path prefix of the Workspace (e.g. '/workspace/{profile_name}/{workspace_name}/')
+	//     - `.PathPrefix`: the path prefix of the Workspace (e.g. '/workspace/connect/{profile_name}/{workspace_name}/')
 	// +kubebuilder:validation:Optional
 	RequestHeaders *IstioHeaderOperations `json:"requestHeaders,omitempty"`
 }

workspaces/controller/cmd/main.go

@@ -20,11 +20,13 @@ import (
 	"crypto/tls"
 	"flag"
 	"os"
+	"strconv"
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
+	istiov1 "istio.io/client-go/pkg/apis/networking/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -36,6 +38,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1"
+	"github.com/kubeflow/notebooks/workspaces/controller/internal/config"
 	controllerInternal "github.com/kubeflow/notebooks/workspaces/controller/internal/controller"
 	"github.com/kubeflow/notebooks/workspaces/controller/internal/helper"
 	webhookInternal "github.com/kubeflow/notebooks/workspaces/controller/internal/webhook"
@@ -50,6 +53,8 @@ var (
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 
+	utilruntime.Must(istiov1.AddToScheme(scheme))
+
 	utilruntime.Must(kubefloworgv1beta1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
@@ -60,6 +65,10 @@ func main() {
 	var probeAddr string
 	var secureMetrics bool
 	var enableHTTP2 bool
+
+	// Define command line flags
+	cfg := &config.EnvConfig{}
+
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
@@ -69,6 +78,15 @@ func main() {
 		"If set the metrics endpoint is served securely")
 	flag.BoolVar(&enableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
+	flag.StringVar(&cfg.IstioGateway, "istio-gateway", getEnvAsStr("ISTIO_GATEWAY", ""),
+		"The name of the Istio gateway to use")
+	flag.StringVar(&cfg.IstioHosts, "istio-hosts", getEnvAsStr("ISTIO_HOSTS", "*"),
+		"The hosts to use for the Istio VirtualService")
+	flag.StringVar(&cfg.ClusterDomain, "cluster-domain", getEnvAsStr("CLUSTER_DOMAIN", "cluster.local"),
+		"The domain to use for the Istio VirtualService")
+	flag.BoolVar(&cfg.UseIstio, "use-istio", getEnvAsBool("USE_ISTIO", false),
+		"If set, Istio will be used")
+
 	opts := zap.Options{
 		Development: true,
 	}
@@ -129,14 +147,15 @@ func main() {
 
 	// setup field indexers on the manager cache. we use these indexes to efficiently
 	// query the cache for things like which Workspaces are using a particular WorkspaceKind
-	if err := helper.SetupManagerFieldIndexers(mgr); err != nil {
+	if err := helper.SetupManagerFieldIndexers(mgr, cfg); err != nil {
 		setupLog.Error(err, "unable to setup field indexers")
 		os.Exit(1)
 	}
 
 	if err = (&controllerInternal.WorkspaceReconciler{
 		Client: mgr.GetClient(),
 		Scheme: mgr.GetScheme(),
+		Config: cfg,
 	}).SetupWithManager(mgr, controller.Options{
 		RateLimiter: helper.BuildRateLimiter(),
 	}); err != nil {
@@ -188,3 +207,19 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func getEnvAsStr(name string, defaultVal string) string {
+	if value, exists := os.LookupEnv(name); exists {
+		return value
+	}
+	return defaultVal
+}
+
+func getEnvAsBool(name string, defaultVal bool) bool {
+	if value, exists := os.LookupEnv(name); exists {
+		if boolValue, err := strconv.ParseBool(value); err == nil {
+			return boolValue
+		}
+	}
+	return defaultVal
+}

workspaces/controller/config/crd/bases/kubeflow.org_workspacekinds.yaml

@@ -3699,7 +3699,7 @@ spec:
                               default: false
                               description: |-
                                 if the path prefix is stripped from incoming HTTP requests
-                                 - if true, the '/workspace/{profile_name}/{workspace_name}/' path prefix
+                                 - if true, the '/workspace/connect/{profile_name}/{workspace_name}/' path prefix
                                    is stripped from incoming requests, the application sees the request
                                    as if it was made to '/...'
                                  - this only works if the application serves RELATIVE URLs for its assets
@@ -3710,7 +3710,7 @@ spec:
                                  - sets the `spec.http[].headers.request` of the Istio VirtualService
                                    https://istio.io/latest/docs/reference/config/networking/virtual-service/#Headers-HeaderOperations
                                  - the following string templates are available:
-                                    - `.PathPrefix`: the path prefix of the Workspace (e.g. '/workspace/{profile_name}/{workspace_name}/')
+                                    - `.PathPrefix`: the path prefix of the Workspace (e.g. '/workspace/connect/{profile_name}/{workspace_name}/')
                               properties:
                                 add:
                                   additionalProperties:

workspaces/controller/config/default/components/istio/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+configMapGenerator:
+- envs:
+  - params.env
+  name: kubeflow-workspaces-istio-config
+
+patches:
+- path: manager_istio_patch.yaml

workspaces/controller/config/default/components/istio/manager_istio_patch.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: workspaces-controller
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        env:
+        - name: USE_ISTIO
+          valueFrom:
+            configMapKeyRef:
+              name: kubeflow-workspaces-istio-config
+              key: USE_ISTIO
+        - name: ISTIO_GATEWAY
+          valueFrom:
+            configMapKeyRef:
+              name: kubeflow-workspaces-istio-config
+              key: ISTIO_GATEWAY
+        - name: ISTIO_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: kubeflow-workspaces-istio-config
+              key: ISTIO_HOST
+        - name: CLUSTER_DOMAIN
+          valueFrom:
+            configMapKeyRef:
+              name: kubeflow-workspaces-istio-config
+              key: CLUSTER_DOMAIN

workspaces/controller/config/default/components/istio/params.env

@@ -0,0 +1,4 @@
+USE_ISTIO=true
+ISTIO_GATEWAY=kubeflow/kubeflow-gateway
+ISTIO_HOST=*
+CLUSTER_DOMAIN=cluster.local

workspaces/controller/config/default/kustomization.yaml

@@ -15,6 +15,7 @@ resources:
 
 components:
 - components/common
+- components/istio
 
 patches:
 # [METRICS] The following patch will enable the metrics endpoint. Ensure that you also protect this endpoint.

workspaces/controller/config/samples/jupyterlab_v1beta1_workspacekind.yaml

@@ -153,7 +153,7 @@ spec:
         httpProxy:
 
           ## if the path prefix is stripped from incoming HTTP requests
-          ##  - if true, the '/workspace/{profile_name}/{workspace_name}/' path prefix
+          ##  - if true, the '/workspace/connect/{profile_name}/{workspace_name}/' path prefix
           ##    is stripped from incoming requests, the application sees the request
           ##    as if it was made to '/...'
           ##  - this only works if the application serves RELATIVE URLs for its assets
@@ -164,7 +164,7 @@ spec:
           ##  - sets the `spec.http[].headers.request` of the Istio VirtualService
           ##    https://istio.io/latest/docs/reference/config/networking/virtual-service/#Headers-HeaderOperations
           ##  - the following string templates are available:
-          ##      - `.PathPrefix`: the path prefix of the Workspace (e.g. '/workspace/{profile_name}/{workspace_name}/')
+          ##      - `.PathPrefix`: the path prefix of the Workspace (e.g. '/workspace/connect/{profile_name}/{workspace_name}/')
           ##
           requestHeaders: {}
             #set: { "X-RStudio-Root-Path": "{{ .PathPrefix }}" } # for RStudio

workspaces/controller/go.mod

@@ -7,6 +7,8 @@ require (
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
 	golang.org/x/time v0.3.0
+	istio.io/api v1.22.8
+	istio.io/client-go v1.22.8
 	k8s.io/api v0.31.0
 	k8s.io/apimachinery v0.31.0
 	k8s.io/client-go v0.31.0
@@ -59,6 +61,7 @@ require (
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

workspaces/controller/go.sum

@@ -9,8 +9,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
 github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k=
-github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
+github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
+github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -155,6 +155,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
+google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -170,6 +172,10 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+istio.io/api v1.22.8 h1:mhkaeFJ13WZ2d6pvL9+exNeQ9UB6HX7e6m+XwO9XoYY=
+istio.io/api v1.22.8/go.mod h1:S3l8LWqNYS9yT+d4bH+jqzH2lMencPkW7SKM1Cu9EyM=
+istio.io/client-go v1.22.8 h1:wojmt220jSbfhpRDsPiflj2nSFTBuYtZNiW9hqKeaWE=
+istio.io/client-go v1.22.8/go.mod h1:noO8SoyMxLwni3w+yGK67aydi2klExjmiqnXyeRS/00=
 k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo=
 k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE=
 k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk=

workspaces/controller/internal/config/environment.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+type EnvConfig struct {
+	IstioGateway  string
+	IstioHosts    string
+	ClusterDomain string
+	UseIstio      bool
+}

workspaces/controller/internal/controller/suite_test.go

@@ -34,15 +34,18 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	istiov1 "istio.io/client-go/pkg/apis/networking/v1"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
-	kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1"
+	"github.com/kubeflow/notebooks/workspaces/controller/internal/config"
 	"github.com/kubeflow/notebooks/workspaces/controller/internal/helper"
+
+	kubefloworgv1beta1 "github.com/kubeflow/notebooks/workspaces/controller/api/v1beta1"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -88,6 +91,8 @@ var _ = BeforeSuite(func() {
 	By("setting up the scheme")
 	err = kubefloworgv1beta1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
+	err = istiov1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
 
 	// +kubebuilder:scaffold:scheme
 
@@ -105,14 +110,21 @@ var _ = BeforeSuite(func() {
 	})
 	Expect(err).NotTo(HaveOccurred())
 
+	envConfig := &config.EnvConfig{
+		// TODO: make true once we install Istio CRDs in EnvTest.
+		//       also create unit tests to ensure VirtualService is created by controller.
+		UseIstio: false,
+	}
+
 	By("setting up the field indexers for the controller manager")
-	err = helper.SetupManagerFieldIndexers(k8sManager)
+	err = helper.SetupManagerFieldIndexers(k8sManager, envConfig)
 	Expect(err).NotTo(HaveOccurred())
 
 	By("setting up the Workspace controller")
 	err = (&WorkspaceReconciler{
 		Client: k8sManager.GetClient(),
 		Scheme: k8sManager.GetScheme(),
+		Config: envConfig,
 	}).SetupWithManager(k8sManager, controller.Options{
 		RateLimiter: helper.BuildRateLimiter(),
 	})