Skip to content

Commit 38388ae

Browse files
muicodermuicoder
committed
Adding association between a Kafka cluster and a notifier
Signed-off-by: muicoder <[email protected]> linkedin#611 Adding WeCom/DingTalk template
1 parent e593345 commit 38388ae

File tree

15 files changed

+213
-28
lines changed

15 files changed

+213
-28
lines changed

‎.github/workflows/action.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
jobs:
2+
redis-operator:
3+
runs-on: ubuntu-latest
4+
steps:
5+
- name: Checkout
6+
uses: actions/checkout@v3
7+
with:
8+
fetch-depth: 0
9+
- name: Set up QEMU
10+
uses: docker/setup-qemu-action@v2
11+
- name: Set up Docker Buildx
12+
uses: docker/setup-buildx-action@v2
13+
- name: Login to DockerHub
14+
uses: docker/login-action@v2
15+
with:
16+
password: ${{ secrets.DOCKERHUB_PASSWORD }}
17+
username: ${{ secrets.DOCKERHUB_USERNAME }}
18+
- name: Build and push
19+
uses: docker/build-push-action@v3
20+
with:
21+
context: .
22+
platforms: linux/amd64,linux/arm64
23+
provenance: false
24+
pull: true
25+
push: true
26+
sbom: false
27+
tags: ${{ secrets.DOCKERHUB_USERNAME }}/burrow:action
28+
name: build
29+
on:
30+
workflow_dispatch:

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: CI
22

3-
on: [push, pull_request]
3+
on: [pull_request]
44

55
jobs:
66
test:

‎Dockerfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
# stage 1: builder
2-
FROM golang:1.20.4-alpine as builder
2+
FROM golang:1.20-alpine as builder
33

44
ENV BURROW_SRC /usr/src/Burrow/
55

6-
RUN apk add --no-cache git curl
6+
RUN apk add --no-cache git curl make
77
COPY . $BURROW_SRC
88
WORKDIR $BURROW_SRC
99

10-
RUN go mod tidy && go build -o /tmp/burrow .
10+
RUN make && go build -trimpath -ldflags "-extldflags -static -w -s" -o /tmp/burrow .
1111

1212
# stage 2: runner
1313
FROM alpine:3.18

‎config/burrow.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ min-distance=1
5858

5959
[notifier.default]
6060
class-name="http"
61+
cluster="local"
6162
url-open="http://someservice.example.com:1467/v1/event"
6263
interval=60
6364
timeout=5

‎config/default-dingtalk-post.tmpl‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{"msgtype": "markdown","markdown": {"title":"Kafka LagChecker", "text": "
2+
{{- $StatusURL := "https://pkg.go.dev/github.com/linkedin/Burrow/core/protocol#StatusConstant"}}
3+
{{- $FormatString := "2006-01-02 15:04:05"}}
4+
# Kafka: {{.Cluster}}
5+
ConsumerGroup: {{.Group}}{{- with .Result.Status}}
6+
{{- if eq . 0}}NotFound{{end}}
7+
{{- if eq . 1}}normal{{end}}
8+
{{- if eq . 2}}lagging{{end}}
9+
{{- if eq . 3}}abnormal{{end}}
10+
{{- end}}
11+
**Status:** Total(Partitions={{.Result.TotalPartitions}},Lag={{.Result.TotalLag}})[{{- with .Result.Status}}
12+
{{- if eq . 0}}NotFound{{end}}
13+
{{- if eq . 1}}{{.}}{{end}}
14+
{{- if eq . 2}}{{.}}{{end}}
15+
{{- if eq . 3}}{{.}}{{end}}
16+
{{- end}}]({{$StatusURL}}){{printf "%.2f" .Result.Complete}}
17+
{{- if .Result.Maxlag|maxlag}}
18+
**MaxLagDetails:**
19+
{{- with .Result.Maxlag}}
20+
{{.Topic}}[{{.Status.String}}](){{printf "%.2f" .Complete}}
21+
{{- if .Owner}}\n\tConsumerHost={{.Owner}}{{end}}
22+
{{- if .ClientID}}\n\tConsumerClientID={{.ClientID}}{{end}}
23+
\tCurrentLag={{.CurrentLag}}
24+
\tPartition={{.Partition}}
25+
{{- end}}
26+
{{- end}}
27+
{{- $TotalErrors := len .Result.Partitions}}
28+
{{- if $TotalErrors}}
29+
### {{$TotalErrors}} partitions have problems
30+
>**CountPartitions:**
31+
{{- range $k,$v := .Result.Partitions|partitioncounts}}
32+
{{- if ne $v 0}}\n\t{{$k}}={{$v}}{{end}}
33+
{{- end}}
34+
**TopicsByStatus:**
35+
{{- range $k,$v := .Result.Partitions|topicsbystatus}}
36+
\t{{$k}}={{$v}}
37+
{{- end}}
38+
**PartitionDetails:**
39+
{{- range .Result.Partitions}}
40+
{{.Topic}}[{{.Status.String}}](){{printf "%.2f" .Complete}}
41+
{{- if .Owner}}\n\tConsumerHost={{.Owner}}{{end}}
42+
{{- if .ClientID}}\n\tConsumerClientID={{.ClientID}}{{end}}
43+
\tCurrentLag={{.CurrentLag}}
44+
\tPartition={{.Partition}}
45+
\tStart={{formattimestamp .Start.Timestamp $FormatString}}
46+
\t\tOffset={{.Start.Offset}}\tLag={{.Start.Lag.Value}}
47+
\tEnd={{formattimestamp .End.Timestamp $FormatString}}
48+
\t\tOffset={{.End.Offset}}\tLag={{.End.Lag.Value}}
49+
{{- end}}
50+
{{- end}}
51+
"
52+
}}

‎config/default-wecom-post.tmpl‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{"msgtype": "markdown","markdown": {"content": "
2+
{{- $StatusURL := "https://pkg.go.dev/github.com/linkedin/Burrow/core/protocol#StatusConstant"}}
3+
{{- $FormatString := "2006-01-02 15:04:05"}}
4+
# Kafka: {{.Cluster}}
5+
ConsumerGroup: {{.Group}}{{- with .Result.Status}}
6+
{{- if eq . 0}}NotFound{{end}}
7+
{{- if eq . 1}}<font color=\"info\">normal</font>{{end}}
8+
{{- if eq . 2}}<font color=\"warning\">lagging</font>{{end}}
9+
{{- if eq . 3}}<font color=\"comment\">abnormal</font>{{end}}
10+
{{- end}}
11+
**Status:** Total(Partitions={{.Result.TotalPartitions}},Lag={{.Result.TotalLag}})[{{- with .Result.Status}}
12+
{{- if eq . 0}}NotFound{{end}}
13+
{{- if eq . 1}}<font color=\"info\">{{.}}</font>{{end}}
14+
{{- if eq . 2}}<font color=\"warning\">{{.}}</font>{{end}}
15+
{{- if eq . 3}}<font color=\"comment\">{{.}}</font>{{end}}
16+
{{- end}}]({{$StatusURL}}){{printf "%.2f" .Result.Complete}}
17+
{{- if .Result.Maxlag|maxlag}}
18+
**MaxLagDetails:**
19+
{{- with .Result.Maxlag}}
20+
{{.Topic}}[{{.Status.String}}](){{printf "%.2f" .Complete}}
21+
{{- if .Owner}}\n\tConsumerHost={{.Owner}}{{end}}
22+
{{- if .ClientID}}\n\tConsumerClientID={{.ClientID}}{{end}}
23+
\tCurrentLag={{.CurrentLag}}
24+
\tPartition={{.Partition}}
25+
{{- end}}
26+
{{- end}}
27+
{{- $TotalErrors := len .Result.Partitions}}
28+
{{- if $TotalErrors}}
29+
### <font color=\"comment\">{{$TotalErrors}} partitions have problems</font>
30+
>**CountPartitions:**
31+
{{- range $k,$v := .Result.Partitions|partitioncounts}}
32+
{{- if ne $v 0}}\n\t{{$k}}={{$v}}{{end}}
33+
{{- end}}
34+
**TopicsByStatus:**
35+
{{- range $k,$v := .Result.Partitions|topicsbystatus}}
36+
\t{{$k}}={{$v}}
37+
{{- end}}
38+
**PartitionDetails:**
39+
{{- range .Result.Partitions}}
40+
{{.Topic}}[{{.Status.String}}](){{printf "%.2f" .Complete}}
41+
{{- if .Owner}}\n\tConsumerHost={{.Owner}}{{end}}
42+
{{- if .ClientID}}\n\tConsumerClientID={{.ClientID}}{{end}}
43+
\tCurrentLag={{.CurrentLag}}
44+
\tPartition={{.Partition}}
45+
\tStart={{formattimestamp .Start.Timestamp $FormatString}}
46+
\t\tOffset={{.Start.Offset}}\tLag={{.Start.Lag.Value}}
47+
\tEnd={{formattimestamp .End.Timestamp $FormatString}}
48+
\t\tOffset={{.End.Offset}}\tLag={{.End.Lag.Value}}
49+
{{- end}}
50+
{{- end}}
51+
"
52+
}}

‎core/internal/helpers/coordinators.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,12 @@ func (m *MockModule) GetName() string {
7474
return args.String(0)
7575
}
7676

77+
// GetCluster mocks the notifier.Module GetCluster func
78+
func (m *MockModule) GetCluster() string {
79+
args := m.Called()
80+
return args.String(0)
81+
}
82+
7783
// GetGroupAllowlist mocks the notifier.Module GetGroupAllowlist func
7884
func (m *MockModule) GetGroupAllowlist() *regexp.Regexp {
7985
args := m.Called()

‎core/internal/httpserver/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ func (hc *Coordinator) configNotifierHTTP(w http.ResponseWriter, r *http.Request
213213
SendClose: viper.GetBool(configRoot + ".send-close"),
214214
ExtraCa: viper.GetString(configRoot + ".extra-ca"),
215215
NoVerify: viper.GetString(configRoot + ".noverify"),
216+
Cluster: viper.GetString(configRoot + ".cluster"),
216217
},
217218
Request: requestInfo,
218219
})
@@ -265,6 +266,7 @@ func (hc *Coordinator) configNotifierEmail(w http.ResponseWriter, r *http.Reques
265266
To: viper.GetString(configRoot + ".to"),
266267
ExtraCa: viper.GetString(configRoot + ".extra-ca"),
267268
NoVerify: viper.GetString(configRoot + ".noverify"),
269+
Cluster: viper.GetString(configRoot + ".cluster"),
268270
},
269271
Request: requestInfo,
270272
})

‎core/internal/httpserver/structs.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ type httpResponseConfigModuleNotifierHTTP struct {
202202
SendClose bool `json:"send-close"`
203203
ExtraCa string `json:"extra-ca"`
204204
NoVerify string `json:"noverify"`
205+
Cluster string `json:"cluster"`
205206
}
206207

207208
type httpResponseConfigModuleNotifierSlack struct {
@@ -238,6 +239,7 @@ type httpResponseConfigModuleNotifierEmail struct {
238239
To string `json:"to"`
239240
ExtraCa string `json:"extra-ca"`
240241
NoVerify string `json:"noverify"`
242+
Cluster string `json:"cluster"`
241243
}
242244

243245
type httpResponseConfigModuleNotifierNull struct {

‎core/internal/notifier/coordinator.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ import (
4848
type Module interface {
4949
protocol.Module
5050
GetName() string
51+
GetCluster() string
5152
GetGroupAllowlist() *regexp.Regexp
5253
GetGroupDenylist() *regexp.Regexp
5354
GetLogger() *zap.Logger
@@ -95,7 +96,7 @@ type Coordinator struct {
9596

9697
// getModuleForClass returns the correct module based on the passed className. As part of the Configure steps, if there
9798
// is any error, it will panic with an appropriate message describing the problem.
98-
func getModuleForClass(app *protocol.ApplicationContext, moduleName, className string, groupAllowlist, groupDenylist *regexp.Regexp, extras map[string]string, templateOpen, templateClose *template.Template) protocol.Module {
99+
func getModuleForClass(app *protocol.ApplicationContext, moduleName, className string, groupAllowlist, groupDenylist *regexp.Regexp, extras map[string]string, templateOpen, templateClose *template.Template, cluster string) protocol.Module {
99100
logger := app.Logger.With(
100101
zap.String("type", "module"),
101102
zap.String("coordinator", "notifier"),
@@ -113,6 +114,7 @@ func getModuleForClass(app *protocol.ApplicationContext, moduleName, className s
113114
extras: extras,
114115
templateOpen: templateOpen,
115116
templateClose: templateClose,
117+
cluster: cluster,
116118
}
117119
case "email":
118120
return &EmailNotifier{
@@ -123,6 +125,7 @@ func getModuleForClass(app *protocol.ApplicationContext, moduleName, className s
123125
extras: extras,
124126
templateOpen: templateOpen,
125127
templateClose: templateClose,
128+
cluster: cluster,
126129
}
127130
case "null":
128131
return &NullNotifier{
@@ -133,6 +136,7 @@ func getModuleForClass(app *protocol.ApplicationContext, moduleName, className s
133136
extras: extras,
134137
templateOpen: templateOpen,
135138
templateClose: templateClose,
139+
cluster: cluster,
136140
}
137141
default:
138142
panic("Unknown notifier className provided: " + className)
@@ -194,6 +198,8 @@ func (nc *Coordinator) Configure() {
194198
groupAllowlist = re
195199
}
196200

201+
cluster := viper.GetString(configRoot + ".cluster")
202+
197203
// Compile the denylist for the consumer groups to not notify for
198204
var groupDenylist *regexp.Regexp
199205
denylist := viper.GetString(configRoot + ".group-denylist")
@@ -227,7 +233,7 @@ func (nc *Coordinator) Configure() {
227233
templateClose = tmpl.Templates()[0]
228234
}
229235

230-
module := getModuleForClass(nc.App, name, viper.GetString(configRoot+".class-name"), groupAllowlist, groupDenylist, extras, templateOpen, templateClose)
236+
module := getModuleForClass(nc.App, name, viper.GetString(configRoot+".class-name"), groupAllowlist, groupDenylist, extras, templateOpen, templateClose, cluster)
231237
module.Configure(name, configRoot)
232238
nc.modules[name] = module
233239
interval := viper.GetInt64(configRoot + ".interval")
@@ -436,6 +442,9 @@ func (nc *Coordinator) checkAndSendResponseToModules(response *protocol.Consumer
436442
for _, genericModule := range nc.modules {
437443
module := genericModule.(Module)
438444

445+
if module.GetCluster() != "" && response.Cluster != module.GetCluster() {
446+
continue
447+
}
439448
// No allowlist means everything passes
440449
groupAllowlist := module.GetGroupAllowlist()
441450
groupDenylist := module.GetGroupDenylist()

0 commit comments

Comments
 (0)