Skip to content

Commit 4bd7dfe

Browse files
ci: add istio to e2e (#646)
* test: implement Istio installation utilities for e2e testing - Add InstallIstioctl() function to download and install istioctl binary - Add InstallIstioMinimalWithIngress() to set up Istio with minimal profile - Add IsIstioInstalled() and WaitIstioctlAvailable() helper functions - Use positional formatting in URL template for istioctl downloads - Support configurable Istio namespace for installation - Add error handling and proper command output redirection This enables e2e tests to automatically set up Istio service mesh components required for workspace HTTP proxy functionality. Signed-off-by: Yash Pal <[email protected]> Signed-off-by: Andy Stoneberg <[email protected]> * ci: add istio to e2e Refactors work submitted by @yashpal2104 to be more aligned with the structure of cert-manager and prometheus. - Note I also rebased the branch off the latest upstream/notebooks-v2 branch. Includes a commented out block of code that adds istio-injection label on the namespaces created by e2e. - VirtualService objects must exist for this code to be uncommented - otherwise the Workspace connect endpoint is unreachable. Moved handling of `istioctl` dependency to the `Makefile` as that code was unwiedly/intimidating to manage in a `.go` file. Additionally, refactored utils module to be decomposed into multiple files as it was becoming unwieldy to manage everything in utils.go. Signed-off-by: Andy Stoneberg <[email protected]> --------- Signed-off-by: Andy Stoneberg <[email protected]> Co-authored-by: Yash Pal <[email protected]>
1 parent 48ae3c7 commit 4bd7dfe

File tree

9 files changed

+609
-260
lines changed

9 files changed

+609
-260
lines changed

workspaces/controller/Makefile

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,12 @@ test: manifests generate fmt vet envtest ## Run tests.
6767
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
6868

6969
# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
70-
# Prometheus and CertManager are installed by default; skip with:
70+
# Prometheus, CertManager, and Istio are installed by default; skip with:
7171
# - PROMETHEUS_INSTALL_SKIP=true
7272
# - CERT_MANAGER_INSTALL_SKIP=true
73+
# - ISTIO_INSTALL_SKIP=true
7374
.PHONY: test-e2e
74-
test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
75+
test-e2e: manifests generate fmt vet istioctl ## Run the e2e tests. Expected an isolated environment using Kind.
7576
@command -v kind >/dev/null 2>&1 || { \
7677
echo "Kind is not installed. Please install Kind manually."; \
7778
exit 1; \
@@ -80,6 +81,7 @@ test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated
8081
echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \
8182
exit 1; \
8283
}
84+
8385
go test ./test/e2e/ -v -ginkgo.v
8486

8587
.PHONY: lint
@@ -157,12 +159,14 @@ KUBECTL ?= kubectl
157159
KUSTOMIZE ?= $(LOCALBIN)/kustomize
158160
CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
159161
ENVTEST ?= $(LOCALBIN)/setup-envtest
162+
ISTIOCTL ?= $(LOCALBIN)/istioctl
160163
GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
161164

162165
## Tool Versions
163166
KUSTOMIZE_VERSION ?= v5.5.0
164167
CONTROLLER_TOOLS_VERSION ?= v0.16.4
165168
ENVTEST_VERSION ?= release-0.19
169+
ISTIOCTL_VERSION ?= 1.27.3
166170
GOLANGCI_LINT_VERSION ?= v1.61.0
167171

168172
.PHONY: kustomize
@@ -180,6 +184,12 @@ envtest: $(ENVTEST) ## Download setup-envtest locally if necessary.
180184
$(ENVTEST): $(LOCALBIN)
181185
$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION))
182186

187+
188+
.PHONY: istioctl
189+
istioctl: $(ISTIOCTL) ## Download istioctl locally if necessary.
190+
$(ISTIOCTL): $(LOCALBIN)
191+
$(call go-install-tool,$(ISTIOCTL),istio.io/istio/istioctl/cmd/istioctl,$(ISTIOCTL_VERSION))
192+
183193
.PHONY: golangci-lint
184194
golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
185195
$(GOLANGCI_LINT): $(LOCALBIN)

workspaces/controller/test/e2e/e2e_suite_test.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ var (
4040

4141
// isPrometheusOperatorAlreadyInstalled will be set true when prometheus CRDs be found on the cluster
4242
// isPrometheusOperatorAlreadyInstalled = false
43+
44+
skipIstioInstall = os.Getenv("ISTIO_INSTALL_SKIP") == "true"
45+
isIstioAlreadyInstalled = false
4346
)
4447

4548
// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated,
@@ -88,6 +91,22 @@ var _ = BeforeSuite(func() {
8891
}
8992
By("checking that cert manager is running")
9093
Expect(utils.WaitCertManagerRunning()).To(Succeed(), "CertManager is not running")
94+
95+
if !skipIstioInstall {
96+
By("checking if istio is installed already")
97+
isIstioAlreadyInstalled = utils.IsIstioCRDsInstalled()
98+
99+
if !isIstioAlreadyInstalled {
100+
_, _ = fmt.Fprintf(GinkgoWriter, "Installing istio...\n")
101+
Expect(utils.InstallIstio()).To(Succeed(), "Failed to install istio")
102+
} else {
103+
_, _ = fmt.Fprintf(GinkgoWriter,
104+
"WARNING: istio is already installed. Skipping installation...\n")
105+
}
106+
107+
By("checking that istio is available")
108+
Expect(utils.WaitIstioAvailable()).To(Succeed(), "istio is not available")
109+
}
91110
})
92111

93112
var _ = AfterSuite(func() {
@@ -103,4 +122,10 @@ var _ = AfterSuite(func() {
103122
_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n")
104123
utils.UninstallCertManager()
105124
}
125+
126+
if !skipIstioInstall && !isIstioAlreadyInstalled {
127+
By("uninstalling Istio")
128+
_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling Istio...\n")
129+
utils.UninstallIstio()
130+
}
106131
})

workspaces/controller/test/e2e/e2e_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,14 @@ var _ = Describe("controller", Ordered, func() {
7676
cmd = exec.Command("kubectl", "create", "ns", workspaceNamespace)
7777
_, _ = utils.Run(cmd) // ignore errors because namespace may already exist
7878

79+
// TODO: enable Istio injection once we have logic to create VirtualServices during Workspace reconciliation
80+
// By("labeling namespaces for Istio injection")
81+
// err := utils.LabelNamespaceForIstioInjection(controllerNamespace)
82+
// ExpectWithOffset(1, err).NotTo(HaveOccurred())
83+
84+
// err = utils.LabelNamespaceForIstioInjection(workspaceNamespace)
85+
// ExpectWithOffset(1, err).NotTo(HaveOccurred())
86+
7987
By("creating common workspace resources")
8088
cmd = exec.Command("kubectl", "apply",
8189
"-k", filepath.Join(projectDir, "config/samples/common"),
Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
Copyright 2024.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package utils
18+
19+
import (
20+
"fmt"
21+
"os/exec"
22+
"strings"
23+
)
24+
25+
const (
26+
// use LTS version of cert-manager
27+
certManagerVersion = "v1.12.13"
28+
certManagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml"
29+
)
30+
31+
// UninstallCertManager uninstalls the cert manager
32+
func UninstallCertManager() {
33+
url := fmt.Sprintf(certManagerURLTmpl, certManagerVersion)
34+
cmd := exec.Command("kubectl", "delete", "-f", url)
35+
if _, err := Run(cmd); err != nil {
36+
warnError(err)
37+
}
38+
}
39+
40+
// InstallCertManager installs the cert manager bundle.
41+
func InstallCertManager() error {
42+
// remove any existing cert-manager leases
43+
// NOTE: this is required to avoid issues where cert-manager is reinstalled quickly due to rerunning tests
44+
cmd := exec.Command("kubectl", "delete",
45+
"leases",
46+
"--ignore-not-found",
47+
"--namespace", "kube-system",
48+
"cert-manager-controller",
49+
"cert-manager-cainjector-leader-election",
50+
)
51+
_, err := Run(cmd)
52+
if err != nil {
53+
return err
54+
}
55+
56+
// install cert-manager
57+
url := fmt.Sprintf(certManagerURLTmpl, certManagerVersion)
58+
cmd = exec.Command("kubectl", "apply", "-f", url)
59+
_, err = Run(cmd)
60+
return err
61+
}
62+
63+
// WaitCertManagerRunning waits for cert manager to be running, and returns an error if not.
64+
func WaitCertManagerRunning() error {
65+
66+
// Wait for the cert-manager Deployments to be Available
67+
cmd := exec.Command("kubectl", "wait",
68+
"deployment.apps",
69+
"--for", "condition=Available",
70+
"--selector", "app.kubernetes.io/instance=cert-manager",
71+
"--all-namespaces",
72+
"--timeout", "5m",
73+
)
74+
_, err := Run(cmd)
75+
if err != nil {
76+
return err
77+
}
78+
79+
// Wait for the cert-manager Endpoints to be ready
80+
// NOTE: the webhooks will not function correctly until this is ready
81+
cmd = exec.Command("kubectl", "wait",
82+
"endpoints",
83+
"--for", "jsonpath=subsets[0].addresses[0].targetRef.kind=Pod",
84+
"--selector", "app.kubernetes.io/instance=cert-manager",
85+
"--all-namespaces",
86+
"--timeout", "2m",
87+
)
88+
_, err = Run(cmd)
89+
return err
90+
}
91+
92+
// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed
93+
// by verifying the existence of key CRDs related to Cert Manager.
94+
func IsCertManagerCRDsInstalled() bool {
95+
// List of common Cert Manager CRDs
96+
certManagerCRDs := []string{
97+
"certificates.cert-manager.io",
98+
"issuers.cert-manager.io",
99+
"clusterissuers.cert-manager.io",
100+
"certificaterequests.cert-manager.io",
101+
"orders.acme.cert-manager.io",
102+
"challenges.acme.cert-manager.io",
103+
}
104+
105+
// Execute the kubectl command to get all CRDs
106+
cmd := exec.Command("kubectl", "get", "crds", "-o", "name")
107+
output, err := Run(cmd)
108+
if err != nil {
109+
return false
110+
}
111+
112+
// Check if any of the Cert Manager CRDs are present
113+
crdList := GetNonEmptyLines(output)
114+
for _, crd := range certManagerCRDs {
115+
for _, line := range crdList {
116+
if strings.Contains(line, crd) {
117+
return true
118+
}
119+
}
120+
}
121+
122+
return false
123+
}
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
Copyright 2024.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package utils
18+
19+
import (
20+
"fmt"
21+
"os"
22+
"os/exec"
23+
"strings"
24+
25+
"github.com/onsi/ginkgo/v2"
26+
)
27+
28+
func warnError(err error) {
29+
_, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "warning: %v\n", err)
30+
}
31+
32+
// Run executes the provided command within this context
33+
func Run(cmd *exec.Cmd) (string, error) {
34+
dir, _ := GetProjectDir()
35+
cmd.Dir = dir
36+
37+
if err := os.Chdir(cmd.Dir); err != nil {
38+
_, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "chdir dir: %s\n", err)
39+
}
40+
41+
cmd.Env = append(os.Environ(), "GO111MODULE=on")
42+
command := strings.Join(cmd.Args, " ")
43+
_, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s\n", command)
44+
output, err := cmd.CombinedOutput()
45+
if err != nil {
46+
return string(output), fmt.Errorf("%s failed with error: (%w) %s", command, err, string(output))
47+
}
48+
49+
return string(output), nil
50+
}
51+
52+
// GetNonEmptyLines converts given command output string into individual objects
53+
// according to line breakers, and ignores the empty elements in it.
54+
func GetNonEmptyLines(output string) []string {
55+
var res []string
56+
elements := strings.Split(output, "\n")
57+
for _, element := range elements {
58+
if element != "" {
59+
res = append(res, element)
60+
}
61+
}
62+
63+
return res
64+
}
65+
66+
// GetProjectDir will return the directory where the project is
67+
func GetProjectDir() (string, error) {
68+
wd, err := os.Getwd()
69+
if err != nil {
70+
return wd, err
71+
}
72+
wd = strings.ReplaceAll(wd, "/test/e2e", "")
73+
return wd, nil
74+
}

0 commit comments

Comments
 (0)