Skip to content

Commit 7bb7ddd

Browse files
committed
feat: add prometheus receiver
1 parent 2668c43 commit 7bb7ddd

File tree

7 files changed

+282
-1
lines changed

7 files changed

+282
-1
lines changed

README.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -521,3 +521,27 @@ receivers:
521521
foo: bar
522522
url: http://127.0.0.1:3100/loki/api/v1/push
523523
```
524+
525+
# Prometheus
526+
527+
The Prometheus receiver emits event count metrics that Prometheus can scrape. Prometheus must
528+
be configured to scrape the kubernetes-event-exporter metrics.
529+
530+
Resource kinds and event reasons must be specified. Metrics will be emitted for only those
531+
resources and their associated events.
532+
533+
```yaml
534+
receivers:
535+
- name: "prometheus"
536+
prometheus:
537+
# Specify a prefix for all event count metrics
538+
eventsMetricsNamePrefix: "metric_prefix_"
539+
# Specify resource kinds and which event reasons to capture for each kind
540+
# Only events with the given reasons for each kind will be emitted
541+
reasonFilter:
542+
Pod:
543+
- "FailedMount"
544+
- "Unhealthy"
545+
Job:
546+
- "DeadlineExceeded"
547+
```

config.example.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,3 +122,9 @@ receivers:
122122
deliveryStreamName: "kubernetes-events"
123123
region: "us-east-1"
124124
deDot: true
125+
- name: "prometheus"
126+
prometheus:
127+
- "Pod"
128+
reasonFilter:
129+
Pod:
130+
- "FailedMount"

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ require (
1818
github.com/prometheus/exporter-toolkit v0.10.0
1919
github.com/rs/zerolog v1.28.0
2020
github.com/slack-go/slack v0.12.0
21-
github.com/stretchr/testify v1.8.1
21+
github.com/stretchr/testify v1.9.0
2222
google.golang.org/api v0.107.0
2323
gopkg.in/natefinch/lumberjack.v2 v2.0.0
2424
k8s.io/api v0.26.7
@@ -98,6 +98,7 @@ require (
9898
github.com/sirupsen/logrus v1.9.0 // indirect
9999
github.com/spf13/cast v1.3.1 // indirect
100100
github.com/spf13/pflag v1.0.5 // indirect
101+
github.com/stretchr/objx v0.5.2 // indirect
101102
go.opencensus.io v0.24.0 // indirect
102103
golang.org/x/crypto v0.17.0 // indirect
103104
golang.org/x/net v0.17.0 // indirect

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
289289
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
290290
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
291291
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
292+
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
293+
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
292294
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
293295
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
294296
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -299,6 +301,8 @@ github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
299301
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
300302
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
301303
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
304+
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
305+
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
302306
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
303307
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
304308
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=

pkg/sinks/prometheus.go

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package sinks
2+
3+
import (
4+
"context"
5+
"strings"
6+
7+
"k8s.io/utils/strings/slices"
8+
9+
"github.com/prometheus/client_golang/prometheus"
10+
"github.com/resmoio/kubernetes-event-exporter/pkg/kube"
11+
"github.com/rs/zerolog/log"
12+
)
13+
14+
func newGaugeVec(opts prometheus.GaugeOpts, labelNames []string) *prometheus.GaugeVec {
15+
v := prometheus.NewGaugeVec(opts, labelNames)
16+
prometheus.MustRegister(v)
17+
return v
18+
}
19+
20+
type PrometheusConfig struct {
21+
EventsMetricsNamePrefix string `yaml:"eventsMetricsNamePrefix"`
22+
ReasonFilter map[string][]string `yaml:"reasonFilter"`
23+
}
24+
25+
type PrometheusGaugeVec interface {
26+
With(labels prometheus.Labels) prometheus.Gauge
27+
Delete(labels prometheus.Labels) bool
28+
}
29+
30+
type PrometheusSink struct {
31+
cfg *PrometheusConfig
32+
kinds []string
33+
metricsByKind map[string]PrometheusGaugeVec
34+
}
35+
36+
func NewPrometheusSink(config *PrometheusConfig) (Sink, error) {
37+
if config.EventsMetricsNamePrefix == "" {
38+
config.EventsMetricsNamePrefix = "event_exporter_"
39+
}
40+
41+
metricsByKind := map[string]PrometheusGaugeVec{}
42+
43+
log.Info().Msgf("Initializing new Prometheus sink...")
44+
kinds := []string{}
45+
for kind := range config.ReasonFilter {
46+
kinds = append(kinds, kind)
47+
metricName := config.EventsMetricsNamePrefix + strings.ToLower(kind) + "_event_count"
48+
metricLabels := []string{strings.ToLower(kind), "namespace", "reason"}
49+
metricsByKind[kind] = newGaugeVec(
50+
prometheus.GaugeOpts{
51+
Name: metricName,
52+
Help: "Event counts for " + kind + " resources.",
53+
}, metricLabels)
54+
55+
log.Info().Msgf("Created metric: %s, will emit events: %v with additional labels: %v", kind, config.ReasonFilter[kind], metricLabels)
56+
}
57+
58+
return &PrometheusSink{
59+
cfg: config,
60+
kinds: kinds,
61+
metricsByKind: metricsByKind,
62+
}, nil
63+
}
64+
65+
func (o *PrometheusSink) Send(ctx context.Context, ev *kube.EnhancedEvent) error {
66+
kind := ev.InvolvedObject.Kind
67+
if slices.Contains(o.kinds, kind) {
68+
for _, reason := range o.cfg.ReasonFilter[kind] {
69+
if ev.Reason == reason {
70+
SetEventCount(o.metricsByKind[kind], ev.InvolvedObject, reason, ev.Count)
71+
} else {
72+
DeleteEventCount(o.metricsByKind[kind], ev.InvolvedObject, reason)
73+
}
74+
}
75+
}
76+
77+
return nil
78+
}
79+
80+
func (o *PrometheusSink) Close() {
81+
// No-op
82+
}
83+
84+
func getMetricLabels(obj kube.EnhancedObjectReference, reason string) prometheus.Labels {
85+
prometheusLabels := prometheus.Labels{
86+
strings.ToLower(obj.Kind): obj.Name,
87+
"namespace": obj.Namespace,
88+
"reason": reason,
89+
}
90+
91+
return prometheusLabels
92+
}
93+
94+
func SetEventCount(metric PrometheusGaugeVec, obj kube.EnhancedObjectReference, reason string, count int32) {
95+
labels := getMetricLabels(obj, reason)
96+
log.Info().Msgf("Setting event count metric with labels: %v", labels)
97+
metric.With(labels).Set(float64(count))
98+
}
99+
100+
func DeleteEventCount(metric PrometheusGaugeVec, obj kube.EnhancedObjectReference, reason string) {
101+
labels := getMetricLabels(obj, reason)
102+
log.Info().Msgf("Deleting event count metric with labels: %v", labels)
103+
metric.Delete(labels)
104+
}

pkg/sinks/prometheus_test.go

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package sinks
2+
3+
import (
4+
"context"
5+
"strings"
6+
"testing"
7+
8+
"github.com/prometheus/client_golang/prometheus"
9+
"github.com/resmoio/kubernetes-event-exporter/pkg/kube"
10+
"github.com/stretchr/testify/mock"
11+
)
12+
13+
type mockGauge struct {
14+
mock.Mock
15+
prometheus.Gauge
16+
}
17+
18+
func (m *mockGauge) Set(count float64) {
19+
m.Called(count)
20+
}
21+
22+
type mockGuageVec struct {
23+
mock.Mock
24+
*prometheus.GaugeVec
25+
}
26+
27+
func (v *mockGuageVec) With(labels prometheus.Labels) prometheus.Gauge {
28+
withArgs := v.Called(labels)
29+
return withArgs.Get(0).(prometheus.Gauge)
30+
}
31+
32+
func (v *mockGuageVec) Delete(labels prometheus.Labels) bool {
33+
deleteArgs := v.Called(labels)
34+
return deleteArgs.Get(0).(bool)
35+
}
36+
37+
func mockEvent(kind string, name string, namespace string, reason string, count int32) *kube.EnhancedEvent {
38+
ev := &kube.EnhancedEvent{}
39+
ev.Reason = reason
40+
ev.Count = count
41+
ev.InvolvedObject.Kind = kind
42+
ev.InvolvedObject.Name = name
43+
ev.InvolvedObject.Namespace = namespace
44+
45+
return ev
46+
}
47+
48+
func TestPrometheusSink_Send(t *testing.T) {
49+
configKind := "Pod"
50+
configReason := "Starting"
51+
testEvent := mockEvent("Pod", "testpod", "testnamespace", "Starting", 1)
52+
53+
tests := []struct {
54+
name string
55+
configKind string
56+
configReason string
57+
ev *kube.EnhancedEvent
58+
wantPrometheusLabels prometheus.Labels
59+
wantErr bool
60+
wantSetCalled bool
61+
wantDeleteCalled bool
62+
}{
63+
{
64+
name: "emits desired resource event with resource label",
65+
configKind: configKind,
66+
configReason: configReason,
67+
ev: testEvent,
68+
wantPrometheusLabels: prometheus.Labels{
69+
strings.ToLower(configKind): testEvent.InvolvedObject.Name,
70+
"namespace": testEvent.InvolvedObject.Namespace,
71+
"reason": configReason,
72+
},
73+
wantErr: false,
74+
wantSetCalled: true,
75+
wantDeleteCalled: false,
76+
},
77+
{
78+
name: "deletes desired resource event with resource label",
79+
configKind: configKind,
80+
configReason: "Creating",
81+
ev: testEvent,
82+
wantPrometheusLabels: prometheus.Labels{
83+
strings.ToLower(configKind): testEvent.InvolvedObject.Name,
84+
"namespace": testEvent.InvolvedObject.Namespace,
85+
"reason": "Creating",
86+
},
87+
wantErr: false,
88+
wantSetCalled: false,
89+
wantDeleteCalled: true,
90+
},
91+
{
92+
name: "does nothing if kind is not expected",
93+
configKind: "ReplicaSet",
94+
configReason: "SuccessfulCreate",
95+
ev: testEvent,
96+
wantPrometheusLabels: prometheus.Labels{},
97+
wantErr: false,
98+
wantSetCalled: false,
99+
wantDeleteCalled: false,
100+
},
101+
}
102+
for _, tt := range tests {
103+
mockGauge := &mockGauge{}
104+
mockGauge.On("Set", mock.Anything).Return()
105+
mockPodMetric := &mockGuageVec{}
106+
mockPodMetric.On("With", mock.Anything).Return(mockGauge)
107+
mockPodMetric.On("Delete", mock.Anything).Return(true)
108+
109+
t.Run(tt.name, func(t *testing.T) {
110+
o := &PrometheusSink{
111+
cfg: &PrometheusConfig{
112+
EventsMetricsNamePrefix: "test_prefix_",
113+
ReasonFilter: map[string][]string{tt.configKind: {tt.configReason}},
114+
},
115+
kinds: []string{tt.configKind},
116+
metricsByKind: map[string]PrometheusGaugeVec{tt.configKind: mockPodMetric},
117+
}
118+
if err := o.Send(context.TODO(), tt.ev); (err != nil) != tt.wantErr {
119+
t.Errorf("PrometheusSink.Send() error = %v, wantErr %v", err, tt.wantErr)
120+
}
121+
122+
if tt.wantSetCalled {
123+
mockPodMetric.AssertCalled(t, "With", tt.wantPrometheusLabels)
124+
mockGauge.AssertCalled(t, "Set", float64(1))
125+
} else {
126+
mockPodMetric.AssertNotCalled(t, "With")
127+
mockGauge.AssertNotCalled(t, "Set")
128+
}
129+
130+
if tt.wantDeleteCalled {
131+
mockPodMetric.AssertCalled(t, "Delete", tt.wantPrometheusLabels)
132+
} else {
133+
mockPodMetric.AssertNotCalled(t, "Delete")
134+
}
135+
})
136+
}
137+
}

pkg/sinks/receiver.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ type ReceiverConfig struct {
2626
BigQuery *BigQueryConfig `yaml:"bigquery"`
2727
EventBridge *EventBridgeConfig `yaml:"eventbridge"`
2828
Pipe *PipeConfig `yaml:"pipe"`
29+
Prometheus *PrometheusConfig `yaml:"prometheus"`
2930
}
3031

3132
func (r *ReceiverConfig) Validate() error {
@@ -122,5 +123,9 @@ func (r *ReceiverConfig) GetSink() (Sink, error) {
122123
return NewLoki(r.Loki)
123124
}
124125

126+
if r.Prometheus != nil {
127+
return NewPrometheusSink(r.Prometheus)
128+
}
129+
125130
return nil, errors.New("unknown sink")
126131
}

0 commit comments

Comments
 (0)