Skip to content

Commit 93f5c58

Browse files
wkingwking
committed
test/extended/cli/adm_upgrade/recommend: Proxy CA for ingress
Avoid [1]: : [Serial][sig-cli] oc adm upgrade recommend When the update service has conditional recommendations runs successfully with an accepted conditional recommendation to the --version target [Suite:openshift/conformance/serial] 20s { fail [github.com/openshift/origin/test/extended/cli/adm_upgrade/recommend.go:225]: Unexpected error: <*errors.errorString | 0xc008463cc0>: expected: Failed to check for at least some preconditions: failed to get alerts from Thanos: unable to get /api/v1/alerts from URI in the openshift-monitoring/thanos-querier Route: thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX->Get "https://thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX/api/v1/alerts": tls: failed to verify certificate: x509: certificate signed by unknown authority by retrieving the default (self-signed) ingress certificate and injecting it into the global Proxy configuration. The default ingress certificate and Proxy configuration knobs I'm using are documented in [2]. The router-certs-default fallback may not be documented, but is backed by [3,4]. [1]: https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/30113/pull-ci-openshift-origin-main-e2e-aws-ovn-serial-2of2/1956090254670696448 [2]: https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/security_and_compliance/configuring-certificates#replacing-default-ingress_replacing-default-ingress [3]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/certificate/default_cert.go#L76-L83 [4]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/names.go#L152-L159
1 parent b18aac0 commit 93f5c58

File tree

1 file changed

+126
-10
lines changed

1 file changed

+126
-10
lines changed

‎test/extended/cli/adm_upgrade/recommend.go

Lines changed: 126 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@ import (
1818
"github.com/openshift/origin/test/extended/util/image"
1919
appsv1 "k8s.io/api/apps/v1"
2020
corev1 "k8s.io/api/core/v1"
21+
"k8s.io/apimachinery/pkg/api/errors"
2122
"k8s.io/apimachinery/pkg/api/resource"
2223
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/apimachinery/pkg/util/wait"
2325
"k8s.io/kubernetes/test/e2e/framework"
2426
)
2527

@@ -30,7 +32,9 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
3032
f := framework.NewDefaultFramework("oc-adm-upgrade-recommend")
3133
oc := exutil.NewCLIWithFramework(f).AsAdmin()
3234
var cv *configv1.ClusterVersion
33-
var restoreChannel, restoreUpstream bool
35+
var proxy *configv1.Proxy
36+
var newProxyCAs string
37+
var restoreChannel, restoreUpstream, restoreProxy bool
3438

3539
g.BeforeAll(func() {
3640
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
@@ -41,6 +45,9 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
4145

4246
cv, err = oc.AdminConfigClient().ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
4347
o.Expect(err).NotTo(o.HaveOccurred())
48+
49+
proxy, err = oc.AdminConfigClient().ConfigV1().Proxies().Get(ctx, "cluster", metav1.GetOptions{})
50+
o.Expect(err).NotTo(o.HaveOccurred())
4451
})
4552

4653
g.AfterAll(func() {
@@ -51,6 +58,18 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
5158
if restoreUpstream {
5259
oc.Run("patch", "clusterversions.config.openshift.io", "version", "--type", "json", "-p", fmt.Sprintf(`[{"op": "add", "path": "/spec/upstream", "value": "%s"}]`, cv.Spec.Upstream)).Execute()
5360
}
61+
62+
if restoreProxy {
63+
if proxy == nil {
64+
oc.AdminConfigClient().ConfigV1().Proxies().Delete(ctx, "cluster", metav1.DeleteOptions{})
65+
} else {
66+
oc.Run("patch", "proxies.config.openshift.io", "cluster", "--type", "json", "-p", fmt.Sprintf(`[{"op": "add", "path": "/spec/trustedCA/name", "value": "%s"}]`, proxy.Spec.TrustedCA.Name)).Execute()
67+
}
68+
}
69+
70+
if newProxyCAs != "" {
71+
oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-config").Delete(ctx, newProxyCAs, metav1.DeleteOptions{})
72+
}
5473
})
5574

5675
g.It("runs successfully, even without upstream OpenShift Update Service customization", func() {
@@ -80,7 +99,11 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
8099
}
81100

82101
graph := fmt.Sprintf(`{"nodes": [{"version": "%s","payload": "%s", "metadata": {"io.openshift.upgrades.graph.release.channels": "test-channel,other-channel"}}]}`, cv.Status.Desired.Version, cv.Status.Desired.Image)
83-
newUpstream, err := runUpdateService(ctx, oc, graph)
102+
newUpstream, newProxyCASecret, err := runUpdateService(ctx, oc, graph, false)
103+
if newProxyCASecret != "" {
104+
restoreProxy = true
105+
newProxyCAs = newProxyCASecret
106+
}
84107
o.Expect(err).NotTo(o.HaveOccurred())
85108

86109
err = oc.Run("adm", "upgrade", "channel", "test-channel").Execute()
@@ -160,7 +183,11 @@ No updates available. You may still upgrade to a specific release image.*`)
160183
o.Expect(err).NotTo(o.HaveOccurred())
161184
graph := buf.String()
162185

163-
newUpstream, err := runUpdateService(ctx, oc, graph)
186+
newUpstream, newProxyCASecret, err := runUpdateService(ctx, oc, graph, true)
187+
if newProxyCASecret != "" {
188+
restoreProxy = true
189+
newProxyCAs = newProxyCASecret
190+
}
164191
o.Expect(err).NotTo(o.HaveOccurred())
165192

166193
err = oc.Run("adm", "upgrade", "channel", "test-channel").Execute()
@@ -183,7 +210,7 @@ No updates available. You may still upgrade to a specific release image.*`)
183210
o.Expect(err).NotTo(o.HaveOccurred())
184211
err = matchRegexp(out, `Upstream update service: http://.*
185212
Channel: test-channel [(]available channels: other-channel, test-channel[)]
186-
213+
FIXME
187214
Updates to 4.[0-9]*:
188215
189216
Version: 4[.][0-9]*[.]0
@@ -216,7 +243,7 @@ Updates to 4[.][0-9]*:
216243
217244
Upstream update service: http://.*
218245
Channel: test-channel [(]available channels: other-channel, test-channel[)]
219-
246+
FIXME
220247
Update to 4[.][0-9]*[.]0 Recommended=False:
221248
Image: example.com/test@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc
222249
Release URL: https://example.com/release/4[.][0-9]*[.]0
@@ -228,7 +255,8 @@ Message: (?s:.*)This is a test risk. https://example.com/testRiskA`)
228255
})
229256
})
230257

231-
func runUpdateService(ctx context.Context, oc *exutil.CLI, graph string) (*url.URL, error) {
258+
func runUpdateService(ctx context.Context, oc *exutil.CLI, graph string, proxyTrustIngress bool) (*url.URL, string, error) {
259+
newProxyCAs := ""
232260
deployment, err := oc.AdminKubeClient().AppsV1().Deployments(oc.Namespace()).Create(ctx,
233261
&appsv1.Deployment{
234262
ObjectMeta: metav1.ObjectMeta{
@@ -275,7 +303,7 @@ python3 -m http.server --bind ::
275303
},
276304
}, metav1.CreateOptions{})
277305
if err != nil {
278-
return nil, err
306+
return nil, newProxyCAs, err
279307
}
280308

281309
service, err := oc.AdminKubeClient().CoreV1().Services(oc.Namespace()).Create(ctx,
@@ -292,16 +320,104 @@ python3 -m http.server --bind ::
292320
},
293321
}, metav1.CreateOptions{})
294322
if err != nil {
295-
return nil, err
323+
return nil, newProxyCAs, err
324+
}
325+
326+
var defaultIngressCert string
327+
if proxyTrustIngress {
328+
defaultIngressSecretName, err := oc.Run("get").Args("--namespace=openshift-ingress-operator", "-o", "jsonpath={.spec.defaultCertificate.name}", "ingresscontroller.operator.openshift.io", "default").Output()
329+
if err != nil {
330+
return nil, newProxyCAs, err
331+
}
332+
333+
if defaultIngressSecretName == "" {
334+
defaultIngressSecretName = "router-certs-default"
335+
}
336+
337+
defaultIngressCert, err = oc.Run("extract").Args("--namespace=openshift-ingress", fmt.Sprintf("secret/%s", defaultIngressSecretName), "--keys=tls.crt", "--to=-").Output()
338+
if err != nil {
339+
return nil, newProxyCAs, err
340+
}
341+
framework.Logf("default ingress certificate: %q", defaultIngressCert)
342+
updatedProxyCAs := defaultIngressCert
343+
344+
proxy, err := oc.AdminConfigClient().ConfigV1().Proxies().Get(ctx, "cluster", metav1.GetOptions{})
345+
if err != nil && errors.IsNotFound(err) {
346+
return nil, newProxyCAs, err
347+
} else if proxy.Spec.TrustedCA.Name != "" {
348+
originalProxyCAs, err := oc.Run("extract").Args("--namespace=openshift-config", fmt.Sprintf("secret/%s", proxy.Spec.TrustedCA.Name), "--keys=ca-bundle.crt", "--to=-").Output()
349+
if err != nil {
350+
return nil, newProxyCAs, err
351+
}
352+
framework.Logf("original proxy CAs: %q", originalProxyCAs)
353+
354+
updatedProxyCAs = fmt.Sprintf("%s%s\n", updatedProxyCAs, originalProxyCAs)
355+
}
356+
357+
configMap, err := oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-config").Create(ctx,
358+
&corev1.ConfigMap{
359+
ObjectMeta: metav1.ObjectMeta{
360+
GenerateName: "test-proxy-and-ingress-cas-",
361+
},
362+
Data: map[string]string{
363+
"ca-bundle.crt": updatedProxyCAs,
364+
},
365+
}, metav1.CreateOptions{})
366+
if err != nil {
367+
return nil, newProxyCAs, err
368+
}
369+
newProxyCAs = configMap.ObjectMeta.Name
370+
371+
if proxy == nil {
372+
proxy, err = oc.AdminConfigClient().ConfigV1().Proxies().Create(ctx,
373+
&configv1.Proxy{
374+
ObjectMeta: metav1.ObjectMeta{
375+
Name: "cluster",
376+
},
377+
Spec: configv1.ProxySpec{
378+
TrustedCA: configv1.ConfigMapNameReference{
379+
Name: newProxyCAs,
380+
},
381+
},
382+
}, metav1.CreateOptions{})
383+
if err != nil {
384+
return nil, newProxyCAs, err
385+
}
386+
} else {
387+
err = oc.Run("patch", "proxies.config.openshift.io", "cluster", "--type", "json", "-p", fmt.Sprintf(`[{"op": "add", "path": "/spec/trustedCA/name", "value": "%s"}]`, newProxyCAs)).Execute()
388+
if err != nil {
389+
return nil, newProxyCAs, err
390+
}
391+
}
296392
}
393+
proxyConfiguredAt := time.Now()
297394

298395
if err = exutil.WaitForDeploymentReady(oc, deployment.ObjectMeta.Name, oc.Namespace(), -1); err != nil {
299-
return nil, err
396+
return nil, newProxyCAs, err
397+
}
398+
399+
if proxyTrustIngress {
400+
if err = wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(_ context.Context) (bool, error) {
401+
trustedCABundle, err := oc.Run("extract").Args("--namespace=openshift-config-managed", "configmap/trusted-ca-bundle", "--keys=ca-bundle.crt", "--to=-").Output()
402+
if err != nil {
403+
return false, err
404+
}
405+
406+
if strings.Contains(trustedCABundle, defaultIngressCert) {
407+
framework.Logf("openshift-config-managed ConfigMap trusted-ca-bundle contains the default ingress certificate after %s", time.Now().Sub(proxyConfiguredAt))
408+
return true, nil
409+
}
410+
411+
framework.Logf("openshift-config-managed ConfigMap trusted-ca-bundle does not yet contain the default ingress certificate, waiting on network operator progagation from the 'cluster' Proxy configuration for %s", time.Now().Sub(proxyConfiguredAt))
412+
return false, nil
413+
}); err != nil {
414+
return nil, newProxyCAs, err
415+
}
300416
}
301417

302418
return &url.URL{
303419
Scheme: "http",
304420
Host: net.JoinHostPort(service.Spec.ClusterIP, strconv.Itoa(int(service.Spec.Ports[0].Port))),
305421
Path: "graph",
306-
}, nil
422+
}, newProxyCAs, nil
307423
}

0 commit comments

Comments
 (0)