Skip to content

Commit 6f8ca39

Browse files
wkingwking
committed
test/extended/cli/adm_upgrade/recommend: Proxy CA for ingress
Avoid [1]: : [Serial][sig-cli] oc adm upgrade recommend When the update service has conditional recommendations runs successfully with an accepted conditional recommendation to the --version target [Suite:openshift/conformance/serial] 20s { fail [github.com/openshift/origin/test/extended/cli/adm_upgrade/recommend.go:225]: Unexpected error: <*errors.errorString | 0xc008463cc0>: expected: Failed to check for at least some preconditions: failed to get alerts from Thanos: unable to get /api/v1/alerts from URI in the openshift-monitoring/thanos-querier Route: thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX->Get "https://thanos-querier-openshift-monitoring.apps.ci-op-p9ttsvlv-173fd.XXXXXXXXXXXXXXXXXXXXXX/api/v1/alerts": tls: failed to verify certificate: x509: certificate signed by unknown authority by retrieving the default (self-signed) ingress certificate and injecting it into the global Proxy configuration. The default ingress certificate and Proxy configuration knobs I'm using are documented in [2]. The router-certs-default fallback may not be documented, but is backed by [3,4]. [1]: https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/30113/pull-ci-openshift-origin-main-e2e-aws-ovn-serial-2of2/1956090254670696448 [2]: https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html/security_and_compliance/configuring-certificates#replacing-default-ingress_replacing-default-ingress [3]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/certificate/default_cert.go#L76-L83 [4]: https://github.com/openshift/cluster-ingress-operator/blob/afb2160975399f4249d9d100641ce32a33c262f1/pkg/operator/controller/names.go#L152-L159
1 parent b18aac0 commit 6f8ca39

File tree

1 file changed

+104
-10
lines changed

1 file changed

+104
-10
lines changed

‎test/extended/cli/adm_upgrade/recommend.go

Lines changed: 104 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"github.com/openshift/origin/test/extended/util/image"
1919
appsv1 "k8s.io/api/apps/v1"
2020
corev1 "k8s.io/api/core/v1"
21+
"k8s.io/apimachinery/pkg/api/errors"
2122
"k8s.io/apimachinery/pkg/api/resource"
2223
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2324
"k8s.io/kubernetes/test/e2e/framework"
@@ -30,7 +31,9 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
3031
f := framework.NewDefaultFramework("oc-adm-upgrade-recommend")
3132
oc := exutil.NewCLIWithFramework(f).AsAdmin()
3233
var cv *configv1.ClusterVersion
33-
var restoreChannel, restoreUpstream bool
34+
var proxy *configv1.Proxy
35+
var newProxyCAs string
36+
var restoreChannel, restoreUpstream, restoreProxy bool
3437

3538
g.BeforeAll(func() {
3639
isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient())
@@ -41,6 +44,9 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
4144

4245
cv, err = oc.AdminConfigClient().ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
4346
o.Expect(err).NotTo(o.HaveOccurred())
47+
48+
proxy, err = oc.AdminConfigClient().ConfigV1().Proxies().Get(ctx, "cluster", metav1.GetOptions{})
49+
o.Expect(err).NotTo(o.HaveOccurred())
4450
})
4551

4652
g.AfterAll(func() {
@@ -51,6 +57,18 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
5157
if restoreUpstream {
5258
oc.Run("patch", "clusterversions.config.openshift.io", "version", "--type", "json", "-p", fmt.Sprintf(`[{"op": "add", "path": "/spec/upstream", "value": "%s"}]`, cv.Spec.Upstream)).Execute()
5359
}
60+
61+
if restoreProxy {
62+
if proxy == nil {
63+
oc.AdminConfigClient().ConfigV1().Proxies().Delete(ctx, "cluster", metav1.DeleteOptions{})
64+
} else {
65+
oc.Run("patch", "proxies.config.openshift.io", "version", "--type", "json", "-p", fmt.Sprintf(`[{"op": "add", "path": "/spec/trustedCA/name", "value": "%s"}]`, proxy.Spec.TrustedCA.Name)).Execute()
66+
}
67+
}
68+
69+
if newProxyCAs != "" {
70+
oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-config").Delete(ctx, newProxyCAs, metav1.DeleteOptions{})
71+
}
5472
})
5573

5674
g.It("runs successfully, even without upstream OpenShift Update Service customization", func() {
@@ -80,7 +98,11 @@ var _ = g.Describe("[Serial][sig-cli] oc adm upgrade recommend", g.Ordered, func
8098
}
8199

82100
graph := fmt.Sprintf(`{"nodes": [{"version": "%s","payload": "%s", "metadata": {"io.openshift.upgrades.graph.release.channels": "test-channel,other-channel"}}]}`, cv.Status.Desired.Version, cv.Status.Desired.Image)
83-
newUpstream, err := runUpdateService(ctx, oc, graph)
101+
newUpstream, newProxyCASecret, err := runUpdateService(ctx, oc, graph, false)
102+
if newProxyCASecret != "" {
103+
restoreProxy = true
104+
newProxyCAs = newProxyCASecret
105+
}
84106
o.Expect(err).NotTo(o.HaveOccurred())
85107

86108
err = oc.Run("adm", "upgrade", "channel", "test-channel").Execute()
@@ -160,7 +182,11 @@ No updates available. You may still upgrade to a specific release image.*`)
160182
o.Expect(err).NotTo(o.HaveOccurred())
161183
graph := buf.String()
162184

163-
newUpstream, err := runUpdateService(ctx, oc, graph)
185+
newUpstream, newProxyCASecret, err := runUpdateService(ctx, oc, graph, true)
186+
if newProxyCASecret != "" {
187+
restoreProxy = true
188+
newProxyCAs = newProxyCASecret
189+
}
164190
o.Expect(err).NotTo(o.HaveOccurred())
165191

166192
err = oc.Run("adm", "upgrade", "channel", "test-channel").Execute()
@@ -183,7 +209,7 @@ No updates available. You may still upgrade to a specific release image.*`)
183209
o.Expect(err).NotTo(o.HaveOccurred())
184210
err = matchRegexp(out, `Upstream update service: http://.*
185211
Channel: test-channel [(]available channels: other-channel, test-channel[)]
186-
212+
FIXME
187213
Updates to 4.[0-9]*:
188214
189215
Version: 4[.][0-9]*[.]0
@@ -216,7 +242,7 @@ Updates to 4[.][0-9]*:
216242
217243
Upstream update service: http://.*
218244
Channel: test-channel [(]available channels: other-channel, test-channel[)]
219-
245+
FIXME
220246
Update to 4[.][0-9]*[.]0 Recommended=False:
221247
Image: example.com/test@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc
222248
Release URL: https://example.com/release/4[.][0-9]*[.]0
@@ -228,7 +254,8 @@ Message: (?s:.*)This is a test risk. https://example.com/testRiskA`)
228254
})
229255
})
230256

231-
func runUpdateService(ctx context.Context, oc *exutil.CLI, graph string) (*url.URL, error) {
257+
func runUpdateService(ctx context.Context, oc *exutil.CLI, graph string, proxyTrustIngress bool) (*url.URL, string, error) {
258+
newProxyCAs := ""
232259
deployment, err := oc.AdminKubeClient().AppsV1().Deployments(oc.Namespace()).Create(ctx,
233260
&appsv1.Deployment{
234261
ObjectMeta: metav1.ObjectMeta{
@@ -275,7 +302,7 @@ python3 -m http.server --bind ::
275302
},
276303
}, metav1.CreateOptions{})
277304
if err != nil {
278-
return nil, err
305+
return nil, newProxyCAs, err
279306
}
280307

281308
service, err := oc.AdminKubeClient().CoreV1().Services(oc.Namespace()).Create(ctx,
@@ -292,16 +319,83 @@ python3 -m http.server --bind ::
292319
},
293320
}, metav1.CreateOptions{})
294321
if err != nil {
295-
return nil, err
322+
return nil, newProxyCAs, err
323+
}
324+
325+
if proxyTrustIngress {
326+
defaultIngressSecretName, err := oc.Run("get").Args("--namespace=openshift-ingress-operator", "-o", "jsonpath={.spec.defaultCertificate.name}", "ingresscontroller.operator.openshift.io", "default").Output()
327+
if err != nil {
328+
return nil, newProxyCAs, err
329+
}
330+
331+
if defaultIngressSecretName == "" {
332+
defaultIngressSecretName = "router-certs-default"
333+
}
334+
335+
defaultIngressCert, err := oc.Run("extract").Args("--namespace=openshift-ingress", fmt.Sprintf("secret/%s", defaultIngressSecretName), "--keys=tls.crt", "--to=-").Output()
336+
if err != nil {
337+
return nil, newProxyCAs, err
338+
}
339+
framework.Logf("default ingress certificate: %q", defaultIngressCert)
340+
updatedProxyCAs := defaultIngressCert
341+
342+
proxy, err := oc.AdminConfigClient().ConfigV1().Proxies().Get(ctx, "cluster", metav1.GetOptions{})
343+
if err != nil && errors.IsNotFound(err) {
344+
return nil, newProxyCAs, err
345+
} else if proxy.Spec.TrustedCA.Name != "" {
346+
originalProxyCAs, err := oc.Run("extract").Args("--namespace=openshift-config", fmt.Sprintf("secret/%s", proxy.Spec.TrustedCA.Name), "--keys=ca-bundle.crt", "--to=-").Output()
347+
if err != nil {
348+
return nil, newProxyCAs, err
349+
}
350+
framework.Logf("original proxy CAs: %q", originalProxyCAs)
351+
352+
updatedProxyCAs = fmt.Sprintf("%s%s\n", updatedProxyCAs, originalProxyCAs)
353+
}
354+
355+
configMap, err := oc.AdminKubeClient().CoreV1().ConfigMap("openshift-config").Create(ctx,
356+
&corev1.ConfigMap{
357+
ObjectMeta: metav1.ObjectMeta{
358+
GenerateName: "test-proxy-and-ingress-cas-",
359+
},
360+
Data: map[string]string{
361+
"ca-bundle.crt": updatedProxyCAs,
362+
},
363+
}, metav1.CreateOptions{})
364+
if err != nil {
365+
return nil, newProxyCAs, err
366+
}
367+
newProxyCAs = configMap.ObjectMeta.Name
368+
369+
if proxy == nil {
370+
proxy, err = oc.AdminConfigClient().ConfigV1().Proxies().Create(ctx,
371+
&configv1.Proxy{
372+
ObjectMeta: metav1.ObjectMeta{
373+
Name: "cluster",
374+
},
375+
Spec: configv1.ProxySpec{
376+
TrustedCA: configv1.ConfigMapNameReference{
377+
Name: newProxyCAs,
378+
},
379+
},
380+
}, metav1.CreateOptions{})
381+
if err != nil {
382+
return nil, newProxyCAs, err
383+
}
384+
} else {
385+
err = oc.Run("patch", "proxies.config.openshift.io", "version", "--type", "json", "-p", fmt.Sprintf(`[{"op": "add", "path": "/spec/trustedCA/name", "value": "%s"}]`, newProxyCAs)).Execute()
386+
if err != nil {
387+
return nil, newProxyCAs, err
388+
}
389+
}
296390
}
297391

298392
if err = exutil.WaitForDeploymentReady(oc, deployment.ObjectMeta.Name, oc.Namespace(), -1); err != nil {
299-
return nil, err
393+
return nil, newProxyCAs, err
300394
}
301395

302396
return &url.URL{
303397
Scheme: "http",
304398
Host: net.JoinHostPort(service.Spec.ClusterIP, strconv.Itoa(int(service.Spec.Ports[0].Port))),
305399
Path: "graph",
306-
}, nil
400+
}, newProxyCAs, nil
307401
}

0 commit comments

Comments
 (0)