Skip to content

Commit 60c65b2

Browse files
LorcanMcVeighgantony
LorcanMcVeigh
and
gantony
committed
[RS-2546] Update operator to deploy waf-http-filter in Enterprise (#4032)
* Start a test for new behaviour * Use privileged PSS for tigera-gateway ns * Enable EnvoyPatchPolicy * Deploy and configure waf-http-filter * Fix codegen * Fix capitalisation * Support custom envoy proxy * Add comment for PSSPrivileged * Check volumes are set as expected --------- Co-authored-by: Antony Guinard <[email protected]>
1 parent fff3815 commit 60c65b2

File tree

4 files changed

+317
-3
lines changed

4 files changed

+317
-3
lines changed

‎hack/gen-versions/enterprise.go.tpl‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ var (
164164
}
165165
{{- end }}
166166
{{ with index .Components "waf-http-filter" }}
167-
ComponentWafHTTPFilter = Component{
167+
ComponentWAFHTTPFilter = Component{
168168
Version: "{{ .Version }}",
169169
Image: "{{ .Image }}",
170170
Registry: "{{ .Registry }}",
@@ -404,6 +404,7 @@ var (
404404
ComponentFluentdWindows,
405405
ComponentGuardian,
406406
ComponentIntrusionDetectionController,
407+
ComponentWAFHTTPFilter,
407408
ComponentSecurityEventWebhooksProcessor,
408409
ComponentKibana,
409410
ComponentManager,

‎pkg/components/enterprise.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,7 @@ var (
351351
ComponentFluentdWindows,
352352
ComponentGuardian,
353353
ComponentIntrusionDetectionController,
354+
ComponentWAFHTTPFilter,
354355
ComponentSecurityEventWebhooksProcessor,
355356
ComponentKibana,
356357
ComponentManager,

‎pkg/render/gateway_api.go‎

Lines changed: 114 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,11 @@ import (
2525
operatorv1 "github.com/tigera/operator/api/v1"
2626
"github.com/tigera/operator/pkg/common"
2727
"github.com/tigera/operator/pkg/components"
28+
"github.com/tigera/operator/pkg/ptr"
2829
rcomp "github.com/tigera/operator/pkg/render/common/components"
2930
rmeta "github.com/tigera/operator/pkg/render/common/meta"
3031
"github.com/tigera/operator/pkg/render/common/secret"
32+
"github.com/tigera/operator/pkg/render/common/securitycontext"
3133
appsv1 "k8s.io/api/apps/v1"
3234
batchv1 "k8s.io/api/batch/v1"
3335
corev1 "k8s.io/api/core/v1"
@@ -341,6 +343,7 @@ type gatewayAPIImplementationComponent struct {
341343
envoyGatewayImage string
342344
envoyProxyImage string
343345
envoyRatelimitImage string
346+
wafHTTPFilterImage string
344347
}
345348

346349
func GatewayAPIImplementationComponent(cfg *GatewayAPIImplementationConfig) Component {
@@ -368,6 +371,10 @@ func (pr *gatewayAPIImplementationComponent) ResolveImages(is *operatorv1.ImageS
368371
if err != nil {
369372
return err
370373
}
374+
pr.wafHTTPFilterImage, err = components.GetReference(components.ComponentWAFHTTPFilter, reg, path, prefix, is)
375+
if err != nil {
376+
return err
377+
}
371378
} else {
372379
pr.envoyGatewayImage, err = components.GetReference(components.ComponentCalicoEnvoyGateway, reg, path, prefix, is)
373380
if err != nil {
@@ -402,7 +409,7 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien
402409
CreateNamespace(
403410
resources.namespace.Name,
404411
pr.cfg.Installation.KubernetesProvider,
405-
PSSBaseline,
412+
PSSPrivileged, // Needed for HostPath volume to write logs to
406413
pr.cfg.Installation.Azure,
407414
),
408415
}
@@ -481,8 +488,9 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien
481488
// "ShutdownManager" and "RateLimit" images.)
482489
envoyGatewayConfig.Provider.Kubernetes.RateLimitDeployment.Pod.ImagePullSecrets = secret.GetReferenceList(pr.cfg.PullSecrets)
483490

484-
// Enable backend APIs.
491+
// Enable extension APIs.
485492
envoyGatewayConfig.ExtensionAPIs.EnableBackend = true
493+
envoyGatewayConfig.ExtensionAPIs.EnableEnvoyPatchPolicy = true
486494

487495
// Rebuild the ConfigMap with those changes.
488496
envoyGatewayConfigMap := resources.envoyGatewayConfigMap.DeepCopyObject().(*corev1.ConfigMap)
@@ -654,6 +662,110 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig(className string,
654662
}
655663
applyEnvoyProxyServiceOverrides(envoyProxy, classSpec.GatewayService)
656664

665+
// Setup WAF HTTP Filter on Enterprise.
666+
if pr.cfg.Installation.Variant == operatorv1.TigeraSecureEnterprise {
667+
// The WAF HTTP filter is not supported when the envoy proxy is deployed as a DaemonSet
668+
// as there is no support for init containers in a DaemonSet.
669+
if envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment != nil {
670+
// Add or update the Init Container to the deployment
671+
wafHTTPFilter := corev1.Container{
672+
Name: "waf-http-filter",
673+
Image: pr.wafHTTPFilterImage,
674+
Args: []string{
675+
"-logFileDirectory",
676+
"/var/log/calico/waf",
677+
"-logFileName",
678+
"waf.log",
679+
"-socketPath",
680+
"/var/run/waf-http-filter/extproc.sock",
681+
},
682+
RestartPolicy: ptr.ToPtr[corev1.ContainerRestartPolicy](corev1.ContainerRestartPolicyAlways),
683+
VolumeMounts: []corev1.VolumeMount{
684+
{
685+
Name: "waf-http-filter",
686+
MountPath: "/var/run/waf-http-filter",
687+
},
688+
{
689+
Name: "var-log-calico",
690+
MountPath: "/var/log/calico",
691+
},
692+
},
693+
SecurityContext: securitycontext.NewRootContext(true),
694+
}
695+
hasWAFHTTPFilter := false
696+
for i, initContainer := range envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.InitContainers {
697+
if initContainer.Name == wafHTTPFilter.Name {
698+
hasWAFHTTPFilter = true
699+
// Handle update
700+
if initContainer.Image != wafHTTPFilter.Image {
701+
envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.InitContainers[i] = wafHTTPFilter
702+
}
703+
}
704+
}
705+
if !hasWAFHTTPFilter {
706+
envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.InitContainers = append(envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.InitContainers, wafHTTPFilter)
707+
}
708+
709+
// Add or update Container volume mount
710+
socketVolumeMount := corev1.VolumeMount{
711+
Name: "waf-http-filter",
712+
MountPath: "/var/run/waf-http-filter",
713+
}
714+
hasSocketVolumeMount := false
715+
for i, volumeMount := range envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Container.VolumeMounts {
716+
if volumeMount.Name == socketVolumeMount.Name {
717+
hasSocketVolumeMount = true
718+
if volumeMount.MountPath != socketVolumeMount.MountPath {
719+
envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Container.VolumeMounts[i] = socketVolumeMount
720+
}
721+
}
722+
}
723+
if !hasSocketVolumeMount {
724+
envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Container.VolumeMounts = append(envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Container.VolumeMounts, socketVolumeMount)
725+
}
726+
727+
// Add or update Pod volumes
728+
logsVolume := corev1.Volume{
729+
VolumeSource: corev1.VolumeSource{
730+
HostPath: &corev1.HostPathVolumeSource{
731+
Path: "/var/log/calico",
732+
Type: ptr.ToPtr(corev1.HostPathDirectoryOrCreate),
733+
},
734+
},
735+
Name: "var-log-calico",
736+
}
737+
socketVolume := corev1.Volume{
738+
VolumeSource: corev1.VolumeSource{
739+
EmptyDir: &corev1.EmptyDirVolumeSource{},
740+
},
741+
Name: "waf-http-filter",
742+
}
743+
hasLogsVolume := false
744+
hasSocketVolume := false
745+
for i, volume := range envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Volumes {
746+
if volume.Name == logsVolume.Name {
747+
hasLogsVolume = true
748+
// Handle update
749+
if volume.VolumeSource.HostPath.Path != logsVolume.VolumeSource.HostPath.Path || volume.VolumeSource.HostPath.Type != logsVolume.VolumeSource.HostPath.Type {
750+
envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Volumes[i] = logsVolume
751+
}
752+
}
753+
if volume.Name == socketVolume.Name {
754+
hasSocketVolume = true
755+
if volume.VolumeSource.EmptyDir != socketVolume.VolumeSource.EmptyDir {
756+
envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Volumes[i] = socketVolume
757+
}
758+
}
759+
}
760+
if !hasLogsVolume {
761+
envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Volumes = append(envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Volumes, logsVolume)
762+
}
763+
if !hasSocketVolume {
764+
envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Volumes = append(envoyProxy.Spec.Provider.Kubernetes.EnvoyDeployment.Pod.Volumes, socketVolume)
765+
}
766+
}
767+
}
768+
657769
return envoyProxy
658770
}
659771

‎pkg/render/gateway_api_test.go‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
envoyapi "github.com/envoyproxy/gateway/api/v1alpha1"
2424
operatorv1 "github.com/tigera/operator/api/v1"
2525
"github.com/tigera/operator/pkg/components"
26+
"github.com/tigera/operator/pkg/ptr"
2627
rtest "github.com/tigera/operator/pkg/render/common/test"
2728
appsv1 "k8s.io/api/apps/v1"
2829
batchv1 "k8s.io/api/batch/v1"
@@ -719,4 +720,203 @@ var _ = Describe("Gateway API rendering tests", func() {
719720
Expect(ep4.Spec.Provider.Kubernetes.EnvoyService.LoadBalancerSourceRanges).To(ConsistOf("182.98.44.55/24"))
720721
Expect(*ep4.Spec.Provider.Kubernetes.EnvoyService.LoadBalancerIP).To(Equal(lbIP))
721722
})
723+
724+
It("should not deploy waf-http-filter for open-source", func() {
725+
installation := &operatorv1.InstallationSpec{
726+
Variant: operatorv1.Calico,
727+
}
728+
gatewayAPI := &operatorv1.GatewayAPI{
729+
Spec: operatorv1.GatewayAPISpec{
730+
GatewayClasses: []operatorv1.GatewayClassSpec{{Name: "tigera-gateway-class"}},
731+
},
732+
}
733+
gatewayComp := GatewayAPIImplementationComponent(&GatewayAPIImplementationConfig{
734+
Installation: installation,
735+
GatewayAPI: gatewayAPI,
736+
})
737+
738+
objsToCreate, _ := gatewayComp.Objects()
739+
proxy, err := rtest.GetResourceOfType[*envoyapi.EnvoyProxy](objsToCreate, "tigera-gateway-class", "tigera-gateway")
740+
Expect(err).NotTo(HaveOccurred())
741+
envoyDeployment := proxy.Spec.Provider.Kubernetes.EnvoyDeployment
742+
Expect(envoyDeployment).ToNot(BeNil())
743+
Expect(envoyDeployment.InitContainers).To(BeNil())
744+
Expect(envoyDeployment.Container).ToNot(BeNil())
745+
Expect(envoyDeployment.Container.VolumeMounts).To(BeNil())
746+
})
747+
748+
It("should deploy waf-http-filter for Enterprise", func() {
749+
installation := &operatorv1.InstallationSpec{
750+
Variant: operatorv1.TigeraSecureEnterprise,
751+
}
752+
gatewayAPI := &operatorv1.GatewayAPI{
753+
Spec: operatorv1.GatewayAPISpec{
754+
GatewayClasses: []operatorv1.GatewayClassSpec{{Name: "tigera-gateway-class"}},
755+
},
756+
}
757+
gatewayComp := GatewayAPIImplementationComponent(&GatewayAPIImplementationConfig{
758+
Installation: installation,
759+
GatewayAPI: gatewayAPI,
760+
})
761+
762+
objsToCreate, _ := gatewayComp.Objects()
763+
proxy, err := rtest.GetResourceOfType[*envoyapi.EnvoyProxy](objsToCreate, "tigera-gateway-class", "tigera-gateway")
764+
Expect(err).NotTo(HaveOccurred())
765+
766+
envoyDeployment := proxy.Spec.Provider.Kubernetes.EnvoyDeployment
767+
Expect(envoyDeployment).ToNot(BeNil())
768+
769+
Expect(envoyDeployment.Pod).ToNot(BeNil())
770+
Expect(envoyDeployment.Pod.Volumes).To(HaveLen(2))
771+
Expect(envoyDeployment.Pod.Volumes[0].Name).To(Equal("var-log-calico"))
772+
Expect(envoyDeployment.Pod.Volumes[0].HostPath.Path).To(Equal("/var/log/calico"))
773+
Expect(envoyDeployment.Pod.Volumes[1].Name).To(Equal("waf-http-filter"))
774+
Expect(envoyDeployment.Pod.Volumes[1].EmptyDir).ToNot(BeNil())
775+
776+
Expect(envoyDeployment.InitContainers[0].Name).To(Equal("waf-http-filter"))
777+
Expect(*envoyDeployment.InitContainers[0].RestartPolicy).To(Equal(corev1.ContainerRestartPolicyAlways))
778+
Expect(envoyDeployment.InitContainers[0].VolumeMounts).To(HaveLen(2))
779+
Expect(envoyDeployment.InitContainers[0].VolumeMounts).To(ContainElements([]corev1.VolumeMount{
780+
{
781+
Name: "waf-http-filter",
782+
MountPath: "/var/run/waf-http-filter",
783+
},
784+
{
785+
Name: "var-log-calico",
786+
MountPath: "/var/log/calico",
787+
},
788+
}))
789+
790+
Expect(envoyDeployment.Container).ToNot(BeNil())
791+
Expect(envoyDeployment.Container.VolumeMounts).To(HaveLen(1))
792+
Expect(envoyDeployment.Container.VolumeMounts).To(ContainElement(corev1.VolumeMount{
793+
Name: "waf-http-filter",
794+
MountPath: "/var/run/waf-http-filter",
795+
}))
796+
})
797+
798+
It("should deploy waf-http-filter for Enterprise when using a custom proxy", func() {
799+
installation := &operatorv1.InstallationSpec{
800+
Variant: operatorv1.TigeraSecureEnterprise,
801+
}
802+
gatewayAPI := &operatorv1.GatewayAPI{
803+
Spec: operatorv1.GatewayAPISpec{
804+
GatewayClasses: []operatorv1.GatewayClassSpec{{
805+
Name: "custom-class",
806+
EnvoyProxyRef: &operatorv1.NamespacedName{
807+
Namespace: "default",
808+
Name: "my-proxy",
809+
},
810+
}},
811+
},
812+
}
813+
envoyProxy := &envoyapi.EnvoyProxy{
814+
TypeMeta: metav1.TypeMeta{
815+
Kind: "EnvoyProxy",
816+
APIVersion: "gateway.envoyproxy.io/v1alpha1",
817+
},
818+
ObjectMeta: metav1.ObjectMeta{
819+
Name: "my-proxy",
820+
Namespace: "default",
821+
},
822+
Spec: envoyapi.EnvoyProxySpec{
823+
Provider: &envoyapi.EnvoyProxyProvider{
824+
Type: envoyapi.ProviderTypeKubernetes,
825+
Kubernetes: &envoyapi.EnvoyProxyKubernetesProvider{
826+
EnvoyDeployment: &envoyapi.KubernetesDeploymentSpec{
827+
InitContainers: []corev1.Container{
828+
{
829+
Name: "some-other-sidecar",
830+
RestartPolicy: ptr.ToPtr[corev1.ContainerRestartPolicy](corev1.ContainerRestartPolicyAlways),
831+
VolumeMounts: []corev1.VolumeMount{
832+
{
833+
Name: "some-other-volume",
834+
MountPath: "/test",
835+
},
836+
},
837+
},
838+
},
839+
Container: &envoyapi.KubernetesContainerSpec{
840+
VolumeMounts: []corev1.VolumeMount{
841+
{
842+
Name: "some-other-volume",
843+
MountPath: "/test",
844+
},
845+
},
846+
},
847+
Pod: &envoyapi.KubernetesPodSpec{
848+
Volumes: []corev1.Volume{
849+
{
850+
Name: "some-other-volume",
851+
VolumeSource: corev1.VolumeSource{
852+
EmptyDir: &corev1.EmptyDirVolumeSource{},
853+
},
854+
},
855+
},
856+
},
857+
},
858+
},
859+
},
860+
},
861+
}
862+
gatewayComp := GatewayAPIImplementationComponent(&GatewayAPIImplementationConfig{
863+
Installation: installation,
864+
GatewayAPI: gatewayAPI,
865+
CustomEnvoyProxies: map[string]*envoyapi.EnvoyProxy{
866+
"custom-class": envoyProxy,
867+
},
868+
})
869+
870+
objsToCreate, _ := gatewayComp.Objects()
871+
872+
// Get the four expected GatewayClasses.
873+
gc, err := rtest.GetResourceOfType[*gapi.GatewayClass](objsToCreate, "custom-class", "tigera-gateway")
874+
Expect(err).NotTo(HaveOccurred())
875+
876+
// Get their four EnvoyProxies.
877+
Expect(gc.Spec.ParametersRef).NotTo(BeNil())
878+
proxy, err := rtest.GetResourceOfType[*envoyapi.EnvoyProxy](objsToCreate, gc.Spec.ParametersRef.Name, string(*gc.Spec.ParametersRef.Namespace))
879+
Expect(err).NotTo(HaveOccurred())
880+
881+
envoyDeployment := proxy.Spec.Provider.Kubernetes.EnvoyDeployment
882+
Expect(envoyDeployment).ToNot(BeNil())
883+
884+
Expect(envoyDeployment.InitContainers).To(HaveLen(2))
885+
Expect(envoyDeployment.InitContainers[0].Name).To(Equal("some-other-sidecar"))
886+
Expect(envoyDeployment.InitContainers[1].Name).To(Equal("waf-http-filter"))
887+
Expect(*envoyDeployment.InitContainers[1].RestartPolicy).To(Equal(corev1.ContainerRestartPolicyAlways))
888+
Expect(envoyDeployment.InitContainers[1].VolumeMounts).To(HaveLen(2))
889+
Expect(envoyDeployment.InitContainers[1].VolumeMounts).To(ContainElements([]corev1.VolumeMount{
890+
{
891+
Name: "waf-http-filter",
892+
MountPath: "/var/run/waf-http-filter",
893+
},
894+
{
895+
Name: "var-log-calico",
896+
MountPath: "/var/log/calico",
897+
},
898+
}))
899+
900+
Expect(envoyDeployment.Container).ToNot(BeNil())
901+
Expect(envoyDeployment.Container.VolumeMounts).To(ContainElements(
902+
corev1.VolumeMount{
903+
Name: "some-other-volume",
904+
MountPath: "/test",
905+
}, corev1.VolumeMount{
906+
Name: "waf-http-filter",
907+
MountPath: "/var/run/waf-http-filter",
908+
},
909+
))
910+
911+
Expect(envoyDeployment.Pod).ToNot(BeNil())
912+
Expect(envoyDeployment.Pod.Volumes).To(HaveLen(3))
913+
Expect(envoyDeployment.Pod.Volumes[0].Name).To(Equal("some-other-volume"))
914+
Expect(envoyDeployment.Pod.Volumes[0].EmptyDir).ToNot(BeNil())
915+
Expect(envoyDeployment.Pod.Volumes[1].Name).To(Equal("var-log-calico"))
916+
Expect(envoyDeployment.Pod.Volumes[1].HostPath.Path).To(Equal("/var/log/calico"))
917+
Expect(envoyDeployment.Pod.Volumes[2].Name).To(Equal("waf-http-filter"))
918+
Expect(envoyDeployment.Pod.Volumes[2].EmptyDir).ToNot(BeNil())
919+
920+
})
921+
722922
})

0 commit comments

Comments
 (0)