From 5f3249bcb34832c7bc1e18355d10b339d7b7951c Mon Sep 17 00:00:00 2001 From: /gray Date: Sat, 27 Jan 2024 13:33:00 +0800 Subject: [PATCH] feat: dae trace (#435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sumire (菫) <151038614+sumire88@users.noreply.github.com> --- .gitignore | 1 + .gitmodules | 3 + .gitmodules.d.mk | 1 + Makefile | 10 +- cmd/trace.go | 72 +++++++++++ control/kern/headers | 2 +- go.mod | 4 +- go.sum | 4 + trace/kallsyms.go | 71 +++++++++++ trace/kern/headers | 1 + trace/kern/trace.c | 241 ++++++++++++++++++++++++++++++++++++ trace/trace.go | 283 +++++++++++++++++++++++++++++++++++++++++++ trace/utils.go | 50 ++++++++ 13 files changed, 738 insertions(+), 5 deletions(-) create mode 100644 cmd/trace.go create mode 100644 trace/kallsyms.go create mode 160000 trace/kern/headers create mode 100644 trace/kern/trace.c create mode 100644 trace/trace.go create mode 100644 trace/utils.go diff --git a/.gitignore b/.gitignore index 55e2e3b..7d42ba9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ outline.json go-mod/ node_modules/ *.log +.build_tags diff --git a/.gitmodules b/.gitmodules index 6025380..1e34e9a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "control/kern/headers"] path = control/kern/headers url = https://github.com/daeuniverse/dae_bpf_headers +[submodule "trace/kern/headers"] + path = trace/kern/headers + url = https://github.com/daeuniverse/dae_bpf_headers diff --git a/.gitmodules.d.mk b/.gitmodules.d.mk index d9e0ec9..9e52742 100644 --- a/.gitmodules.d.mk +++ b/.gitmodules.d.mk @@ -1 +1,2 @@ submodule_paths=control/kern/headers +submodule_paths=trace/kern/headers diff --git a/Makefile b/Makefile index ecc3487..0614895 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ MAX_MATCH_SET_LEN ?= 64 CFLAGS := -DMAX_MATCH_SET_LEN=$(MAX_MATCH_SET_LEN) $(CFLAGS) NOSTRIP ?= n STRIP_PATH := $(shell command -v $(STRIP) 2>/dev/null) +BUILD_TAGS_FILE := .build_tags ifeq ($(strip $(NOSTRIP)),y) STRIP_FLAG := -no-strip else ifeq ($(wildcard $(STRIP_PATH)),) @@ -47,7 +48,7 @@ dae: export CGO_ENABLED=0 endif dae: ebpf @echo $(CFLAGS) - go build -o $(OUTPUT) $(BUILD_ARGS) . + go build -tags=$(shell cat $(BUILD_TAGS_FILE)) -o $(OUTPUT) $(BUILD_ARGS) . ## End Dae Build ## Begin Git Submodules @@ -74,6 +75,8 @@ submodule submodules: $(submodule_paths) clean-ebpf: @rm -f control/bpf_bpf*.go && \ rm -f control/bpf_bpf*.o + @rm -f trace/bpf_bpf*.go && \ + rm -f trace/bpf_bpf*.o fmt: go fmt ./... @@ -82,10 +85,13 @@ ebpf: export BPF_CLANG := $(CLANG) ebpf: export BPF_STRIP_FLAG := $(STRIP_FLAG) ebpf: export BPF_CFLAGS := $(CFLAGS) ebpf: export BPF_TARGET := $(TARGET) +ebpf: export BPF_TRACE_TARGET := $(GOARCH) ebpf: submodule clean-ebpf @unset GOOS && \ unset GOARCH && \ unset GOARM && \ echo $(STRIP_FLAG) && \ - go generate ./control/control.go + go generate ./control/control.go && \ + go generate ./trace/trace.go && echo trace > $(BUILD_TAGS_FILE) || echo > $(BUILD_TAGS_FILE) + ## End Ebpf diff --git a/cmd/trace.go b/cmd/trace.go new file mode 100644 index 0000000..613c1ad --- /dev/null +++ b/cmd/trace.go @@ -0,0 +1,72 @@ +//go:build trace +// +build trace + +/* + * SPDX-License-Identifier: AGPL-3.0-only + * Copyright (c) 2022-2024, daeuniverse Organization + */ + +package cmd + +import ( + "context" + "os/signal" + "syscall" + + "github.com/daeuniverse/dae/cmd/internal" + "github.com/daeuniverse/dae/trace" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var ( + IPv4, IPv6 bool + L4Proto string + Port int + OutputFile string +) + +func init() { + traceCmd := &cobra.Command{ + Use: "trace", + Short: "To trace traffic", + Run: func(cmd *cobra.Command, args []string) { + internal.AutoSu() + + if IPv4 && IPv6 { + logrus.Fatalln("IPv4 and IPv6 cannot be set at the same time") + } + if !IPv4 && !IPv6 { + IPv4 = true + } + IPVersion := 4 + if IPv6 { + IPVersion = 6 + } + + var L4ProtoNo uint16 + switch L4Proto { + case "tcp": + L4ProtoNo = syscall.IPPROTO_TCP + case "udp": + L4ProtoNo = syscall.IPPROTO_UDP + default: + logrus.Fatalf("Unknown L4 protocol: %s\n", L4Proto) + } + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + if err := trace.StartTrace(ctx, IPVersion, L4ProtoNo, Port, OutputFile); err != nil { + logrus.Fatalln(err) + } + }, + } + + traceCmd.PersistentFlags().BoolVarP(&IPv4, "ipv4", "4", false, "Capture IPv4 traffic") + traceCmd.PersistentFlags().BoolVarP(&IPv6, "ipv6", "6", false, "Capture IPv6 traffic") + traceCmd.PersistentFlags().StringVarP(&L4Proto, "l4-proto", "p", "tcp", "Layer 4 protocol") + traceCmd.PersistentFlags().IntVarP(&Port, "port", "P", 80, "Port") + traceCmd.PersistentFlags().StringVarP(&OutputFile, "output", "o", "/dev/stdout", "Output file") + + rootCmd.AddCommand(traceCmd) +} diff --git a/control/kern/headers b/control/kern/headers index d72c67e..e4da1c9 160000 --- a/control/kern/headers +++ b/control/kern/headers @@ -1 +1 @@ -Subproject commit d72c67ed8f5a7d11774b5cd88734e2ffe6847721 +Subproject commit e4da1c9601e1c3797d02c481a462b66588477495 diff --git a/go.mod b/go.mod index 604fe8b..5ed5365 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/adrg/xdg v0.4.0 github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df github.com/bits-and-blooms/bloom/v3 v3.5.0 - github.com/cilium/ebpf v0.11.0 + github.com/cilium/ebpf v0.12.3 github.com/daeuniverse/dae-config-dist/go/dae_config v0.0.0-20230604120805-1c27619b592d github.com/daeuniverse/outbound v0.0.0-20240101085641-7932e7df927d github.com/daeuniverse/softwind v0.0.0-20231230065827-eed67f20d2c1 @@ -24,7 +24,7 @@ require ( github.com/x-cray/logrus-prefixed-formatter v0.5.2 golang.org/x/crypto v0.12.0 golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 - golang.org/x/sys v0.11.0 + golang.org/x/sys v0.14.1-0.20231108175955-e4099bfacb8c google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) diff --git a/go.sum b/go.sum index 3588dc5..be1a57a 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWk github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= +github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= +github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/daeuniverse/dae-config-dist/go/dae_config v0.0.0-20230604120805-1c27619b592d h1:hnC39MjR7xt5kZjrKlef7DXKFDkiX8MIcDXYC/6Jf9Q= github.com/daeuniverse/dae-config-dist/go/dae_config v0.0.0-20230604120805-1c27619b592d/go.mod h1:VGWGgv7pCP5WGyHGUyb9+nq/gW0yBm+i/GfCNATOJ1M= @@ -200,6 +202,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.1-0.20231108175955-e4099bfacb8c h1:3kC/TjQ+xzIblQv39bCOyRk8fbEeJcDHwbyxPUU2BpA= +golang.org/x/sys v0.14.1-0.20231108175955-e4099bfacb8c/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/trace/kallsyms.go b/trace/kallsyms.go new file mode 100644 index 0000000..aa575d9 --- /dev/null +++ b/trace/kallsyms.go @@ -0,0 +1,71 @@ +/* + * SPDX-License-Identifier: AGPL-3.0-only + * Copyright (c) 2022-2024, daeuniverse Organization + */ + +package trace + +import ( + "bufio" + "os" + "sort" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "golang.org/x/exp/slices" +) + +type Symbol struct { + Type string + Name string + Addr uint64 +} + +var kallsyms []Symbol +var kallsymsByName map[string]Symbol = make(map[string]Symbol) +var kallsymsByAddr map[uint64]Symbol = make(map[uint64]Symbol) + +func init() { + readKallsyms() +} + +func readKallsyms() { + file, err := os.Open("/proc/kallsyms") + if err != nil { + logrus.Fatalf("failed to open /proc/kallsyms: %v", err) + } + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) < 3 { + continue + } + addr, err := strconv.ParseUint(parts[0], 16, 64) + if err != nil { + continue + } + typ, name := parts[1], parts[2] + kallsyms = append(kallsyms, Symbol{typ, name, addr}) + kallsymsByName[name] = Symbol{typ, name, addr} + kallsymsByAddr[addr] = Symbol{typ, name, addr} + } + sort.Slice(kallsyms, func(i, j int) bool { + return kallsyms[i].Addr < kallsyms[j].Addr + }) +} + +func NearestSymbol(addr uint64) Symbol { + idx, _ := slices.BinarySearchFunc(kallsyms, addr, func(x Symbol, addr uint64) int { return int(x.Addr - addr) }) + if idx == len(kallsyms) { + return kallsyms[idx-1] + } + if kallsyms[idx].Addr == addr { + return kallsyms[idx] + } + if idx == 0 { + return kallsyms[0] + } + return kallsyms[idx-1] +} diff --git a/trace/kern/headers b/trace/kern/headers new file mode 160000 index 0000000..e4da1c9 --- /dev/null +++ b/trace/kern/headers @@ -0,0 +1 @@ +Subproject commit e4da1c9601e1c3797d02c481a462b66588477495 diff --git a/trace/kern/trace.c b/trace/kern/trace.c new file mode 100644 index 0000000..bff1d33 --- /dev/null +++ b/trace/kern/trace.c @@ -0,0 +1,241 @@ +#include "headers/if_ether_defs.h" +#include "headers/vmlinux.h" + +#include "headers/bpf_core_read.h" +#include "headers/bpf_endian.h" +#include "headers/bpf_helpers.h" +#include "headers/bpf_tracing.h" + +#define IFNAMSIZ 16 +#define PNAME_LEN 32 + +static const bool TRUE = true; + +union addr { + u32 v4addr; + struct { + u64 d1; + u64 d2; + } v6addr; +} __attribute__((packed)); + +struct meta { + u64 pc; + u64 skb; + u64 second_param; + u32 mark; + u32 netns; + u32 ifindex; + u32 pid; + unsigned char ifname[IFNAMSIZ]; + unsigned char pname[PNAME_LEN]; +} __attribute__((packed)); + +struct tuple { + union addr saddr; + union addr daddr; + u16 sport; + u16 dport; + u16 l3_proto; + u8 l4_proto; + u8 tcp_flags; + u16 payload_len; +} __attribute__((packed)); + +struct event { + struct meta meta; + struct tuple tuple; +} __attribute__((packed)); + +const struct event *_ __attribute__((unused)); + +struct tracing_config { + u16 port; + u16 l4_proto; + u8 ip_vsn; +}; + +static volatile const struct tracing_config tracing_cfg; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, __u64); + __type(value, bool); + __uint(max_entries, 1024); +} skb_addresses SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1<<29); +} events SEC(".maps"); + +static __always_inline u32 +get_netns(struct sk_buff *skb) +{ + u32 netns = BPF_CORE_READ(skb, dev, nd_net.net, ns.inum); + + // if skb->dev is not initialized, try to get ns from sk->__sk_common.skc_net.net->ns.inum + if (netns == 0) { + struct sock *sk = BPF_CORE_READ(skb, sk); + if (sk != NULL) + netns = BPF_CORE_READ(sk, __sk_common.skc_net.net, ns.inum); + } + + return netns; +} + +static __always_inline bool +filter_l3_and_l4(struct sk_buff *skb) +{ + void *skb_head = BPF_CORE_READ(skb, head); + u16 l3_off = BPF_CORE_READ(skb, network_header); + u16 l4_off = BPF_CORE_READ(skb, transport_header); + + struct iphdr *l3_hdr = (struct iphdr *) (skb_head + l3_off); + u8 ip_vsn = BPF_CORE_READ_BITFIELD_PROBED(l3_hdr, version); + if (ip_vsn != tracing_cfg.ip_vsn) + return false; + + u16 l4_proto; + if (ip_vsn == 4) { + struct iphdr *ip4 = (struct iphdr *) l3_hdr; + l4_proto = BPF_CORE_READ(ip4, protocol); + } else if (ip_vsn == 6) { + struct ipv6hdr *ip6 = (struct ipv6hdr *) l3_hdr; + l4_proto = BPF_CORE_READ(ip6, nexthdr); + } else { + return false; + } + + if (l4_proto != tracing_cfg.l4_proto) + return false; + + u16 sport, dport; + if (l4_proto == IPPROTO_TCP) { + struct tcphdr *tcp = (struct tcphdr *) (skb_head + l4_off); + sport = BPF_CORE_READ(tcp, source); + dport = BPF_CORE_READ(tcp, dest); + } else if (l4_proto == IPPROTO_UDP) { + struct udphdr *udp = (struct udphdr *) (skb_head + l4_off); + sport = BPF_CORE_READ(udp, source); + dport = BPF_CORE_READ(udp, dest); + } else { + return false; + } + + if (dport != tracing_cfg.port && sport != tracing_cfg.port) + return false; + + return true; +} + +static __always_inline void +set_meta(struct meta *meta, struct sk_buff *skb, struct pt_regs *ctx) +{ + meta->pc = BPF_CORE_READ(ctx, ip); + meta->skb = (__u64)skb; + meta->second_param = PT_REGS_PARM2(ctx); + meta->mark = BPF_CORE_READ(skb, mark); + meta->netns = get_netns(skb); + meta->ifindex = BPF_CORE_READ(skb, dev, ifindex); + BPF_CORE_READ_STR_INTO(&meta->ifname, skb, dev, name); + + struct task_struct *current = (void *)bpf_get_current_task(); + meta->pid = BPF_CORE_READ(current, pid); + u64 arg_start = BPF_CORE_READ(current, mm, arg_start); + bpf_probe_read_user_str(&meta->pname, PNAME_LEN, (void *)arg_start); +} + +static __always_inline void +set_tuple(struct tuple *tpl, struct sk_buff *skb) +{ + void *skb_head = BPF_CORE_READ(skb, head); + u16 l3_off = BPF_CORE_READ(skb, network_header); + u16 l4_off = BPF_CORE_READ(skb, transport_header); + + struct iphdr *l3_hdr = (struct iphdr *) (skb_head + l3_off); + u8 ip_vsn = BPF_CORE_READ_BITFIELD_PROBED(l3_hdr, version); + + u16 l3_total_len; + if (ip_vsn == 4) { + struct iphdr *ip4 = (struct iphdr *) l3_hdr; + BPF_CORE_READ_INTO(&tpl->saddr, ip4, saddr); + BPF_CORE_READ_INTO(&tpl->daddr, ip4, daddr); + tpl->l4_proto = BPF_CORE_READ(ip4, protocol); + tpl->l3_proto = ETH_P_IP; + l3_total_len = bpf_ntohs(BPF_CORE_READ(ip4, tot_len)); + } else if (ip_vsn == 6) { + struct ipv6hdr *ip6 = (struct ipv6hdr *) l3_hdr; + BPF_CORE_READ_INTO(&tpl->saddr, ip6, saddr); + BPF_CORE_READ_INTO(&tpl->daddr, ip6, daddr); + tpl->l4_proto = BPF_CORE_READ(ip6, nexthdr); + tpl->l3_proto = ETH_P_IPV6; + l3_total_len = bpf_ntohs(BPF_CORE_READ(ip6, payload_len)); + } + u16 l3_hdr_len = l4_off - l3_off; + + u16 l4_hdr_len; + if (tpl->l4_proto == IPPROTO_TCP) { + struct tcphdr *tcp = (struct tcphdr *) (skb_head + l4_off); + tpl->sport= BPF_CORE_READ(tcp, source); + tpl->dport= BPF_CORE_READ(tcp, dest); + bpf_probe_read_kernel(&tpl->tcp_flags, sizeof(tpl->tcp_flags), + (void *)tcp + offsetof(struct tcphdr, ack_seq) + 5); + l4_hdr_len = BPF_CORE_READ_BITFIELD_PROBED(tcp, doff) * 4; + tpl->payload_len = l3_total_len - l3_hdr_len - l4_hdr_len; + } else if (tpl->l4_proto == IPPROTO_UDP) { + struct udphdr *udp = (struct udphdr *) (skb_head + l4_off); + tpl->sport= BPF_CORE_READ(udp, source); + tpl->dport= BPF_CORE_READ(udp, dest); + tpl->payload_len = bpf_ntohs(BPF_CORE_READ(udp, len)) - sizeof(struct udphdr); + } +} + +static __always_inline int +handle_skb(struct sk_buff *skb, struct pt_regs *ctx) +{ + bool tracked = false; + u64 skb_addr = (u64) skb; + struct event ev = {}; + if (bpf_map_lookup_elem(&skb_addresses, &skb_addr)) { + tracked = true; + goto cont; + } + + if (!filter_l3_and_l4(skb)) + return 0; + + if (!tracked) + bpf_map_update_elem(&skb_addresses, &skb_addr, &TRUE, BPF_ANY); + +cont: + set_meta(&ev.meta, skb, ctx); + set_tuple(&ev.tuple, skb); + + bpf_ringbuf_output(&events, &ev, sizeof(ev), 0); + return 0; +} + +#define KPROBE_SKB_AT(X) \ + SEC("kprobe/skb-" #X) \ + int kprobe_skb_##X(struct pt_regs *ctx) \ + { \ + struct sk_buff *skb = (struct sk_buff *) PT_REGS_PARM##X(ctx); \ + return handle_skb(skb, ctx); \ + } + +KPROBE_SKB_AT(1) +KPROBE_SKB_AT(2) +KPROBE_SKB_AT(3) +KPROBE_SKB_AT(4) +KPROBE_SKB_AT(5) + +SEC("kprobe/skb_lifetime_termination") +int kprobe_skb_lifetime_termination(struct pt_regs *ctx) +{ + u64 skb = (u64) PT_REGS_PARM1(ctx); + bpf_map_delete_elem(&skb_addresses, &skb); + return 0; +} + +SEC("license") const char __license[] = "Dual BSD/GPL"; diff --git a/trace/trace.go b/trace/trace.go new file mode 100644 index 0000000..2ee3da2 --- /dev/null +++ b/trace/trace.go @@ -0,0 +1,283 @@ +/* + * SPDX-License-Identifier: AGPL-3.0-only + * Copyright (c) 2022-2024, daeuniverse Organization + */ + +package trace + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "net" + "os" + "syscall" + "unsafe" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/btf" + "github.com/cilium/ebpf/link" + "github.com/cilium/ebpf/ringbuf" + "github.com/sirupsen/logrus" +) + +//go:generate go run -mod=mod github.com/cilium/ebpf/cmd/bpf2go -cc "$BPF_CLANG" "$BPF_STRIP_FLAG" -cflags "$BPF_CFLAGS" -target "$BPF_TRACE_TARGET" -type event bpf kern/trace.c -- -I./headers + +var nativeEndian binary.ByteOrder + +func init() { + buf := [2]byte{} + *(*uint16)(unsafe.Pointer(&buf[0])) = uint16(0xABCD) + + switch buf { + case [2]byte{0xCD, 0xAB}: + nativeEndian = binary.LittleEndian + case [2]byte{0xAB, 0xCD}: + nativeEndian = binary.BigEndian + default: + panic("Could not determine native endianness.") + } +} + +func StartTrace(ctx context.Context, ipVersion int, l4ProtoNo uint16, port int, outputFile string) (err error) { + objs, err := rewriteAndLoadBpf(ipVersion, l4ProtoNo, port) + if err != nil { + return + } + defer objs.Close() + + targets, kfreeSkbReasons, err := searchAvailableTargets() + if err != nil { + return + } + + links, err := attachBpfToTargets(objs, targets) + if err != nil { + return + } + defer func() { + i := 0 + fmt.Printf("\n") + for _, link := range links { + i++ + fmt.Printf("detaching kprobes: %04d/%04d\r", i, len(links)) + link.Close() + } + fmt.Printf("\n") + }() + + fmt.Printf("\nstart tracing\n") + if err = handleEvents(ctx, objs, outputFile, kfreeSkbReasons); err != nil { + return + } + return +} + +func rewriteAndLoadBpf(ipVersion int, l4ProtoNo uint16, port int) (_ *bpfObjects, err error) { + spec, err := loadBpf() + if err != nil { + return nil, fmt.Errorf("failed to load BPF: %+v\n", err) + } + if err := spec.RewriteConstants(map[string]interface{}{ + "tracing_cfg": struct { + port uint16 + l4Proto uint16 + ipVersion uint8 + pad uint8 + }{ + port: Htons(uint16(port)), + l4Proto: uint16(l4ProtoNo), + ipVersion: uint8(ipVersion), + pad: 0, + }, + }); err != nil { + return nil, fmt.Errorf("failed to rewrite constants: %+v\n", err) + } + var opts ebpf.CollectionOptions + opts.Programs.LogLevel = ebpf.LogLevelInstruction + opts.Programs.LogSize = ebpf.DefaultVerifierLogSize * 100 + objs := bpfObjects{} + if err := spec.LoadAndAssign(&objs, &opts); err != nil { + var ( + ve *ebpf.VerifierError + verifierLog string + ) + if errors.As(err, &ve) { + verifierLog = fmt.Sprintf("Verifier error: %+v\n", ve) + } + return nil, fmt.Errorf("failed to load BPF: %+v\n%s", err, verifierLog) + } + + return &objs, nil +} + +func searchAvailableTargets() (targets map[string]int, kfreeSkbReasons map[uint64]string, err error) { + targets = map[string]int{} + + btfSpec, err := btf.LoadKernelSpec() + if err != nil { + return nil, nil, fmt.Errorf("failed to load kernel BTF: %+v\n", err) + } + + if kfreeSkbReasons, err = getKFreeSKBReasons(btfSpec); err != nil { + return + } + + iter := btfSpec.Iterate() + for iter.Next() { + typ := iter.Type + fn, ok := typ.(*btf.Func) + if !ok { + continue + } + + fnName := string(fn.Name) + + fnProto := fn.Type.(*btf.FuncProto) + i := 1 + for _, p := range fnProto.Params { + if ptr, ok := p.Type.(*btf.Pointer); ok { + if strct, ok := ptr.Target.(*btf.Struct); ok { + if strct.Name == "sk_buff" && i <= 5 { + name := fnName + targets[name] = i + continue + } + } + } + i += 1 + } + } + + return targets, kfreeSkbReasons, nil +} + +func getKFreeSKBReasons(spec *btf.Spec) (map[uint64]string, error) { + if _, err := spec.AnyTypeByName("kfree_skb_reason"); err != nil { + // Kernel is too old to have kfree_skb_reason + return nil, nil + } + + var dropReasonsEnum *btf.Enum + if err := spec.TypeByName("skb_drop_reason", &dropReasonsEnum); err != nil { + return nil, fmt.Errorf("failed to find 'skb_drop_reason' enum: %v", err) + } + + ret := map[uint64]string{} + for _, val := range dropReasonsEnum.Values { + ret[uint64(val.Value)] = val.Name + + } + + return ret, nil +} + +func attachBpfToTargets(objs *bpfObjects, targets map[string]int) (links []link.Link, err error) { + kp, err := link.Kprobe("kfree_skbmem", objs.KprobeSkbLifetimeTermination, nil) + if err != nil { + logrus.Warnf("failed to attach kprobe to kfree_skbmem: %+v\n", err) + } + + i := 0 + for fn, pos := range targets { + i++ + fmt.Printf("attaching kprobes: %04d/%04d\r", i, len(targets)) + var kp link.Link + switch pos { + case 1: + kp, err = link.Kprobe(fn, objs.KprobeSkb1, nil) + case 2: + kp, err = link.Kprobe(fn, objs.KprobeSkb2, nil) + case 3: + kp, err = link.Kprobe(fn, objs.KprobeSkb3, nil) + case 4: + kp, err = link.Kprobe(fn, objs.KprobeSkb4, nil) + case 5: + kp, err = link.Kprobe(fn, objs.KprobeSkb5, nil) + } + if err != nil { + logrus.Debugf("failed to attach kprobe to %s: %+v\n", fn, err) + continue + } + links = append(links, kp) + } + if len(links) == 0 { + err = fmt.Errorf("failed to attach kprobes to any target") + } + links = append(links, kp) + return links, nil +} + +func handleEvents(ctx context.Context, objs *bpfObjects, outputFile string, kfreeSkbReasons map[uint64]string) (err error) { + writer, err := os.Create(outputFile) + if err != nil { + return + } + + eventsReader, err := ringbuf.NewReader(objs.Events) + if err != nil { + return fmt.Errorf("failed to create ringbuf reader: %+v\n", err) + } + defer eventsReader.Close() + + go func() { + <-ctx.Done() + eventsReader.Close() + }() + + type bpfEvent struct { + Pc uint64 + Skb uint64 + SecondParam uint64 + Mark uint32 + Netns uint32 + Ifindex uint32 + Pid uint32 + Ifname [16]uint8 + Pname [32]uint8 + Saddr [16]byte + Daddr [16]byte + Sport uint16 + Dport uint16 + L3Proto uint16 + L4Proto uint8 + TcpFlags uint8 + PayloadLen uint16 + } + + for { + rec, err := eventsReader.Read() + if err != nil { + if errors.Is(err, ringbuf.ErrClosed) { + return nil + } + logrus.Debugf("failed to read ringbuf: %+v", err) + continue + } + + var event bpfEvent + if err = binary.Read(bytes.NewBuffer(rec.RawSample), nativeEndian, &event); err != nil { + logrus.Debugf("failed to parse ringbuf event: %+v", err) + continue + } + + fmt.Fprintf(writer, "%x mark=%x netns=%010d if=%d(%s) proc=%d(%s) ", event.Skb, event.Mark, event.Netns, event.Ifindex, TrimNull(string(event.Ifname[:])), event.Pid, TrimNull(string(event.Pname[:]))) + if event.L3Proto == syscall.ETH_P_IP { + fmt.Fprintf(writer, "%s:%d > %s:%d ", net.IP(event.Saddr[:4]).String(), Ntohs(event.Sport), net.IP(event.Daddr[:4]).String(), Ntohs(event.Dport)) + } else { + fmt.Fprintf(writer, "[%s]:%d > [%s]:%d ", net.IP(event.Saddr[:]).String(), Ntohs(event.Sport), net.IP(event.Daddr[:]).String(), Ntohs(event.Dport)) + } + if event.L4Proto == syscall.IPPROTO_TCP { + fmt.Fprintf(writer, "tcp_flags=%s ", TcpFlags(event.TcpFlags)) + } + fmt.Fprintf(writer, "payload_len=%d ", event.PayloadLen) + sym := NearestSymbol(event.Pc) + fmt.Fprintf(writer, "%s", sym.Name) + if sym.Name == "kfree_skb_reason" { + fmt.Fprintf(writer, "(%s)", kfreeSkbReasons[event.SecondParam]) + } + fmt.Fprintf(writer, "\n") + } +} diff --git a/trace/utils.go b/trace/utils.go new file mode 100644 index 0000000..034e2c4 --- /dev/null +++ b/trace/utils.go @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: AGPL-3.0-only + * Copyright (c) 2022-2024, daeuniverse Organization + */ + +package trace + +import ( + "encoding/binary" + "strings" +) + +func Htons(x uint16) uint16 { + data := make([]byte, 2) + nativeEndian.PutUint16(data, x) + return binary.BigEndian.Uint16(data) +} + +func Ntohs(x uint16) uint16 { + data := make([]byte, 2) + binary.BigEndian.PutUint16(data, x) + return nativeEndian.Uint16(data) +} + +func TrimNull(s string) string { + return strings.TrimRight(s, "\x00") +} + +func TcpFlags(data uint8) string { + flags := []string{} + if data&0b00100000 != 0 { + flags = append(flags, "U") + } + if data&0b00010000 != 0 { + flags = append(flags, ".") + } + if data&0b00001000 != 0 { + flags = append(flags, "P") + } + if data&0b00000100 != 0 { + flags = append(flags, "R") + } + if data&0b00000010 != 0 { + flags = append(flags, "S") + } + if data&0b00000001 != 0 { + flags = append(flags, "F") + } + return strings.Join(flags, "") +}