Skip to content

Commit 85fd6ae

Browse files
kmesh-botkmesh-bot
authored
Merge pull request #874 from tacslon/feat/bpf-multi-ver
Build 2 series of eBPF objects(kernel ver. <5.13 & >=5.13) and load eBPF dynamically when Kmesh starts up
2 parents 1ed5a3f + e816990 commit 85fd6ae

File tree

10 files changed

+148
-43
lines changed

10 files changed

+148
-43
lines changed

‎bpf/include/bpf_common.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#define __KMESH_BPF_COMMON_H__
66

77
#include "common.h"
8+
#include "inner_map_defs.h"
89

910
#define map_of_manager kmesh_manage
1011
#define MAP_SIZE_OF_MANAGER 8192
@@ -21,7 +22,6 @@
2122
#define BPF_DATA_MAX_LEN \
2223
192 /* this value should be \
2324
small that make compile success */
24-
#define BPF_INNER_MAP_DATA_LEN 1300
2525

2626
struct manager_key {
2727
union {

‎bpf/include/inner_map_defs.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
2+
/* Copyright Authors of Kmesh */
3+
4+
#ifndef __INNER_MAP_H__
5+
#define __INNER_MAP_H__
6+
7+
#define BPF_INNER_MAP_DATA_LEN 1300
8+
9+
#endif // __INNER_MAP_H__

‎bpf/kmesh/ads/include/kmesh_common.h‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "bpf_common.h"
1010
#include "config.h"
1111
#include "core/address.pb-c.h"
12+
#include "tail_call_index.h"
1213

1314
#define BPF_LOGTYPE_LISTENER BPF_DEBUG_ON
1415
#define BPF_LOGTYPE_FILTERCHAIN BPF_DEBUG_ON
@@ -69,15 +70,6 @@ static inline char *bpf_strncpy(char *dst, int n, const char *src)
6970
}
7071
#endif
7172

72-
typedef enum {
73-
KMESH_TAIL_CALL_LISTENER = 1,
74-
KMESH_TAIL_CALL_FILTER_CHAIN,
75-
KMESH_TAIL_CALL_FILTER,
76-
KMESH_TAIL_CALL_ROUTER,
77-
KMESH_TAIL_CALL_CLUSTER,
78-
KMESH_TAIL_CALL_ROUTER_CONFIG,
79-
} tail_call_index_t;
80-
8173
typedef Core__SocketAddress address_t;
8274

8375
// bpf return value

‎bpf/kmesh/ads/include/tail_call_index.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
2+
/* Copyright Authors of Kmesh */
3+
4+
#ifndef _TAIL_CALL_INDEX_H_
5+
#define _TAIL_CALL_INDEX_H_
6+
7+
typedef enum {
8+
KMESH_TAIL_CALL_LISTENER = 1,
9+
KMESH_TAIL_CALL_FILTER_CHAIN,
10+
KMESH_TAIL_CALL_FILTER,
11+
KMESH_TAIL_CALL_ROUTER,
12+
KMESH_TAIL_CALL_CLUSTER,
13+
KMESH_TAIL_CALL_ROUTER_CONFIG,
14+
} tail_call_index_t;
15+
16+
#endif // _TAIL_CALL_INDEX_H_

‎bpf/kmesh/bpf2go/bpf2go.go‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,18 @@
1818
package bpf2go
1919

2020
// go run github.com/cilium/ebpf/cmd/bpf2go --help
21-
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshCgroupSock ../ads/cgroup_sock.c -- -I../ads/include -I../../include -I../../../api/v2-c -DCGROUP_SOCK_MANAGE
22-
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshCgroupSockWorkload ../workload/cgroup_sock.c -- -I../workload/include -I../../include -I../probes
23-
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSockops ../ads/sockops.c -- -I../ads/include -I../../include -I../../../api/v2-c
24-
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshTracePoint ../ads/tracepoint.c -- -I../ads/include -I../../include
25-
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSockopsWorkload ../workload/sockops.c -- -I../workload/include -I../../include -I../probes
26-
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshXDPAuth ../workload/xdp.c -- -I../workload/include -I../../include -I../../../api/v2-c
27-
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSendmsg ../workload/sendmsg.c -- -I../workload/include -I../../include
21+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshCgroupSock ../ads/cgroup_sock.c -- -I../ads/include -I../../include -I../../../api/v2-c -DCGROUP_SOCK_MANAGE -DKERNEL_VERSION_HIGHER_5_13_0=1
22+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshCgroupSockWorkload ../workload/cgroup_sock.c -- -I../workload/include -I../../include -I../probes -DKERNEL_VERSION_HIGHER_5_13_0=1
23+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSockops ../ads/sockops.c -- -I../ads/include -I../../include -I../../../api/v2-c -DKERNEL_VERSION_HIGHER_5_13_0=1
24+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshTracePoint ../ads/tracepoint.c -- -I../ads/include -I../../include -DKERNEL_VERSION_HIGHER_5_13_0=1
25+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSockopsWorkload ../workload/sockops.c -- -I../workload/include -I../../include -I../probes -DKERNEL_VERSION_HIGHER_5_13_0=1
26+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshXDPAuth ../workload/xdp.c -- -I../workload/include -I../../include -I../../../api/v2-c -DKERNEL_VERSION_HIGHER_5_13_0=1
27+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSendmsg ../workload/sendmsg.c -- -I../workload/include -I../../include -DKERNEL_VERSION_HIGHER_5_13_0=1
28+
29+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshCgroupSockCompat ../ads/cgroup_sock.c -- -I../ads/include -I../../include -I../../../api/v2-c -DCGROUP_SOCK_MANAGE -DKERNEL_VERSION_HIGHER_5_13_0=0
30+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshCgroupSockWorkloadCompat ../workload/cgroup_sock.c -- -I../workload/include -I../../include -I../probes -DKERNEL_VERSION_HIGHER_5_13_0=0
31+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSockopsCompat ../ads/sockops.c -- -I../ads/include -I../../include -I../../../api/v2-c -DKERNEL_VERSION_HIGHER_5_13_0=0
32+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshTracePointCompat ../ads/tracepoint.c -- -I../ads/include -I../../include -DKERNEL_VERSION_HIGHER_5_13_0=0
33+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSockopsWorkloadCompat ../workload/sockops.c -- -I../workload/include -I../../include -I../probes -DKERNEL_VERSION_HIGHER_5_13_0=0
34+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshXDPAuthCompat ../workload/xdp.c -- -I../workload/include -I../../include -I../../../api/v2-c -DKERNEL_VERSION_HIGHER_5_13_0=0
35+
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSendmsgCompat ../workload/sendmsg.c -- -I../workload/include -I../../include -DKERNEL_VERSION_HIGHER_5_13_0=0

‎config/kmesh_marcos_def.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,3 @@
8181
* is enabled accordingly.
8282
* */
8383
#define LIBBPF_HIGHER_0_6_0_VERSION 0
84-
85-
/*
86-
87-
*/
88-
#define KERNEL_VERSION_HIGHER_5_13_0 0

‎kmesh_macros_env.sh‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,3 @@ if [[ "$LIBBPF_VERSION" < "0.6.0" ]]; then
6161
else
6262
set_config LIBBPF_HIGHER_0_6_0_VERSION 1
6363
fi
64-
65-
if [[ "$KERNEL_VERSION" < "5.13.0" ]]; then
66-
set_config KERNEL_VERSION_HIGHER_5_13_0 0
67-
else
68-
set_config KERNEL_VERSION_HIGHER_5_13_0 1
69-
fi

‎pkg/bpf/bpf_kmesh_common.go‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,8 @@
2020
package bpf
2121

2222
// #cgo pkg-config: bpf api-v2-c
23-
// #include "kmesh/ads/include/kmesh_common.h"
23+
// #include "kmesh/ads/include/tail_call_index.h"
24+
// #include "inner_map_defs.h"
2425
import "C"
2526
import (
2627
"os"
@@ -32,6 +33,7 @@ import (
3233

3334
"kmesh.net/kmesh/bpf/kmesh/bpf2go"
3435
"kmesh.net/kmesh/daemon/options"
36+
"kmesh.net/kmesh/pkg/utils"
3537
)
3638

3739
var KMESH_TAIL_CALL_LISTENER = uint32(C.KMESH_TAIL_CALL_LISTENER)
@@ -94,9 +96,12 @@ func (sc *BpfSockConn) loadKmeshSockConnObjects() (*ebpf.CollectionSpec, error)
9496
)
9597
opts.Maps.PinPath = sc.Info.MapPath
9698

97-
spec, err = bpf2go.LoadKmeshCgroupSock()
98-
99-
if err != nil || spec == nil {
99+
if utils.KernelVersionLowerThan5_13() {
100+
spec, err = bpf2go.LoadKmeshCgroupSockCompat()
101+
} else {
102+
spec, err = bpf2go.LoadKmeshCgroupSock()
103+
}
104+
if err != nil {
100105
return nil, err
101106
}
102107

‎pkg/bpf/bpf_kmesh_workload.go‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"kmesh.net/kmesh/bpf/kmesh/bpf2go"
3333
"kmesh.net/kmesh/daemon/options"
3434
"kmesh.net/kmesh/pkg/constants"
35+
"kmesh.net/kmesh/pkg/utils"
3536
)
3637

3738
type BpfSockConnWorkload struct {
@@ -70,14 +71,21 @@ func (sc *BpfSockConnWorkload) loadKmeshSockConnObjects() (*ebpf.CollectionSpec,
7071
opts ebpf.CollectionOptions
7172
)
7273
opts.Maps.PinPath = sc.Info.MapPath
73-
spec, err = bpf2go.LoadKmeshCgroupSockWorkload()
7474

75-
if err != nil || spec == nil {
75+
if utils.KernelVersionLowerThan5_13() {
76+
spec, err = bpf2go.LoadKmeshCgroupSockWorkloadCompat()
77+
} else {
78+
spec, err = bpf2go.LoadKmeshCgroupSockWorkload()
79+
}
80+
if err != nil {
7681
return nil, err
7782
}
7883

7984
SetInnerMap(spec)
8085
setMapPinType(spec, ebpf.PinByName)
86+
// The real difference is in the .o file. The prog and map structures of the structure are exactly the same,
87+
// and the .o file has been loaded into the spec in bpf2go.LoadXX(),so when assigning spec to KmeshCgroupSockWorkloadObjects
88+
// here, there is no need to distinguish the type.
8189
if err = spec.LoadAndAssign(&sc.KmeshCgroupSockWorkloadObjects, &opts); err != nil {
8290
return nil, err
8391
}
@@ -251,13 +259,14 @@ func (so *BpfSockOpsWorkload) loadKmeshSockopsObjects() (*ebpf.CollectionSpec, e
251259

252260
opts.Maps.PinPath = so.Info.MapPath
253261

254-
spec, err = bpf2go.LoadKmeshSockopsWorkload()
262+
if utils.KernelVersionLowerThan5_13() {
263+
spec, err = bpf2go.LoadKmeshSockopsWorkloadCompat()
264+
} else {
265+
spec, err = bpf2go.LoadKmeshSockopsWorkload()
266+
}
255267
if err != nil {
256268
return nil, err
257269
}
258-
if spec == nil {
259-
return nil, fmt.Errorf("error: loadKmeshSockopsObjects() spec is nil")
260-
}
261270

262271
SetInnerMap(spec)
263272
setMapPinType(spec, ebpf.PinByName)
@@ -382,7 +391,13 @@ func (sm *BpfSendMsgWorkload) loadKmeshSendmsgObjects() (*ebpf.CollectionSpec, e
382391
)
383392

384393
opts.Maps.PinPath = sm.Info.MapPath
385-
if spec, err = bpf2go.LoadKmeshSendmsg(); err != nil {
394+
395+
if utils.KernelVersionLowerThan5_13() {
396+
spec, err = bpf2go.LoadKmeshSendmsgCompat()
397+
} else {
398+
spec, err = bpf2go.LoadKmeshSendmsg()
399+
}
400+
if err != nil {
386401
return nil, err
387402
}
388403

@@ -518,13 +533,15 @@ func (xa *BpfXdpAuthWorkload) loadKmeshXdpAuthObjects() (*ebpf.CollectionSpec, e
518533
)
519534

520535
opts.Maps.PinPath = xa.Info.MapPath
521-
spec, err = bpf2go.LoadKmeshXDPAuth()
536+
537+
if utils.KernelVersionLowerThan5_13() {
538+
spec, err = bpf2go.LoadKmeshXDPAuthCompat()
539+
} else {
540+
spec, err = bpf2go.LoadKmeshXDPAuth()
541+
}
522542
if err != nil {
523543
return nil, err
524544
}
525-
if spec == nil {
526-
return nil, fmt.Errorf("error: loadKmeshXdpAuthObjects() spec is nil")
527-
}
528545

529546
SetInnerMap(spec)
530547
setMapPinType(spec, ebpf.PinByName)

‎pkg/utils/kernel_version.go‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
//go:build linux
2+
// +build linux
3+
4+
/*
5+
* Copyright The Kmesh Authors.
6+
*
7+
* Licensed under the Apache License, Version 2.0 (the "License");
8+
* you may not use this file except in compliance with the License.
9+
* You may obtain a copy of the License at:
10+
*
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
*
13+
* Unless required by applicable law or agreed to in writing, software
14+
* distributed under the License is distributed on an "AS IS" BASIS,
15+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
* See the License for the specific language governing permissions and
17+
* limitations under the License.
18+
*/
19+
package utils
20+
21+
import (
22+
"strconv"
23+
"strings"
24+
"syscall"
25+
)
26+
27+
// KernelVersionLowerThan5_13 return whether the current kernel version is lower than 5.13,
28+
// and will fallback to less BPF log ability(return true) if error
29+
func KernelVersionLowerThan5_13() bool {
30+
kernelVersion := GetKernelVersion()
31+
if len(kernelVersion) == 0 {
32+
return true
33+
}
34+
splitVers := strings.Split(kernelVersion, ".")
35+
if len(splitVers) < 2 {
36+
return true
37+
}
38+
39+
mainVer, err := strconv.Atoi(splitVers[0])
40+
if err != nil || mainVer < 5 {
41+
return true
42+
}
43+
if mainVer > 5 {
44+
return false
45+
}
46+
47+
subVer, err := strconv.Atoi(splitVers[1])
48+
return err != nil || subVer < 13
49+
}
50+
51+
// GetKernelVersion return part of the result of 'uname -a' like '5.15.153.1-xxxx'
52+
func GetKernelVersion() string {
53+
var uname syscall.Utsname
54+
if err := syscall.Uname(&uname); err != nil {
55+
return ""
56+
}
57+
return int8ToStr(uname.Release[:])
58+
}
59+
60+
func int8ToStr(arr []int8) string {
61+
b := make([]byte, 0, len(arr))
62+
for _, v := range arr {
63+
if v == 0x00 {
64+
break
65+
}
66+
b = append(b, byte(v))
67+
}
68+
return string(b)
69+
}

0 commit comments

Comments
 (0)