/* * SPDX-License-Identifier: AGPL-3.0-only * Copyright (c) since 2023, mzz2017 */ package control import ( "fmt" "github.com/cilium/ebpf" ciliumLink "github.com/cilium/ebpf/link" "github.com/safchain/ethtool" "github.com/sirupsen/logrus" "github.com/v2rayA/dae/common/consts" internal "github.com/v2rayA/dae/pkg/ebpf_internal" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" "net" "os" "regexp" ) type ControlPlaneCore struct { log *logrus.Logger deferFuncs []func() error bpf *bpfObjects kernelVersion *internal.Version } func (c *ControlPlaneCore) Close() (err error) { // Invoke defer funcs in reverse order. for i := len(c.deferFuncs) - 1; i >= 0; i-- { if e := c.deferFuncs[i](); e != nil { // Combine errors. if err != nil { err = fmt.Errorf("%w; %v", err, e) } else { err = e } } } return err } func getIfParamsFromLink(link netlink.Link) (ifParams bpfIfParams, err error) { // Get link offload features. et, err := ethtool.NewEthtool() if err != nil { return bpfIfParams{}, err } defer et.Close() features, err := et.Features(link.Attrs().Name) if err != nil { return bpfIfParams{}, err } if features["tx-checksum-ip-generic"] { ifParams.TxL4CksmIp4Offload = true ifParams.TxL4CksmIp6Offload = true } if features["tx-checksum-ipv4"] { ifParams.TxL4CksmIp4Offload = true } if features["tx-checksum-ipv6"] { ifParams.TxL4CksmIp6Offload = true } if features["rx-checksum"] { ifParams.RxCksmOffload = true } switch { case regexp.MustCompile(`^docker\d+$`).MatchString(link.Attrs().Name): ifParams.UseNonstandardOffloadAlgorithm = true default: } return ifParams, nil } func (c *ControlPlaneCore) addQdisc(ifname string) error { link, err := netlink.LinkByName(ifname) if err != nil { return err } qdisc := &netlink.GenericQdisc{ QdiscAttrs: netlink.QdiscAttrs{ LinkIndex: link.Attrs().Index, Handle: netlink.MakeHandle(0xffff, 0), Parent: netlink.HANDLE_CLSACT, }, QdiscType: "clsact", } if err := netlink.QdiscAdd(qdisc); err != nil { return fmt.Errorf("cannot add clsact qdisc: %w", err) } return nil } func (c *ControlPlaneCore) delQdisc(ifname string) error { link, err := netlink.LinkByName(ifname) if err != nil { return err } qdisc := &netlink.GenericQdisc{ QdiscAttrs: netlink.QdiscAttrs{ LinkIndex: link.Attrs().Index, Handle: netlink.MakeHandle(0xffff, 0), Parent: netlink.HANDLE_CLSACT, }, QdiscType: "clsact", } if err := netlink.QdiscDel(qdisc); err != nil { if !os.IsExist(err) { return fmt.Errorf("cannot add clsact qdisc: %w", err) } } return nil } func (c *ControlPlaneCore) setupRoutingPolicy() (err error) { /// Insert ip rule / ip route. const table = 2023 /** ip table ip route add local default dev lo table 2023 ip -6 route add local default dev lo table 2023 */ routes := []netlink.Route{{ Scope: unix.RT_SCOPE_HOST, LinkIndex: consts.LoopbackIfIndex, Dst: &net.IPNet{ IP: []byte{0, 0, 0, 0}, Mask: net.CIDRMask(0, 32), }, Table: table, Type: unix.RTN_LOCAL, }, { Scope: unix.RT_SCOPE_HOST, LinkIndex: consts.LoopbackIfIndex, Dst: &net.IPNet{ IP: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, Mask: net.CIDRMask(0, 128), }, Table: table, Type: unix.RTN_LOCAL, }} cleanRoutes := func() error { var errs error for _, route := range routes { if e := netlink.RouteDel(&route); e != nil { if errs != nil { errs = fmt.Errorf("%w; %v", errs, e) } else { errs = e } } } if errs != nil { return fmt.Errorf("IpRouteDel(lo): %w", errs) } return nil } tryRouteAddAgain: for _, route := range routes { if err = netlink.RouteAdd(&route); err != nil { if os.IsExist(err) { _ = cleanRoutes() goto tryRouteAddAgain } return fmt.Errorf("IpRouteAdd: %w", err) } } c.deferFuncs = append(c.deferFuncs, cleanRoutes) /** ip rule ip rule add fwmark 0x8000000/0x8000000 table 2023 ip -6 rule add fwmark 0x8000000/0x8000000 table 2023 */ rules := []netlink.Rule{{ SuppressIfgroup: -1, SuppressPrefixlen: -1, Priority: -1, Goto: -1, Flow: -1, Family: unix.AF_INET, Table: table, Mark: int(consts.TproxyMark), Mask: int(consts.TproxyMark), }, { SuppressIfgroup: -1, SuppressPrefixlen: -1, Priority: -1, Goto: -1, Flow: -1, Family: unix.AF_INET6, Table: table, Mark: int(consts.TproxyMark), Mask: int(consts.TproxyMark), }} cleanRules := func() error { var errs error for _, rule := range rules { if e := netlink.RuleDel(&rule); e != nil { if errs != nil { errs = fmt.Errorf("%w; %v", errs, e) } else { errs = e } } } if errs != nil { return fmt.Errorf("IpRuleDel: %w", errs) } return nil } tryRuleAddAgain: for _, rule := range rules { if err = netlink.RuleAdd(&rule); err != nil { if os.IsExist(err) { _ = cleanRules() goto tryRuleAddAgain } return fmt.Errorf("IpRuleAdd: %w", err) } } c.deferFuncs = append(c.deferFuncs, cleanRules) return nil } func (c *ControlPlaneCore) bindLan(ifname string) error { c.log.Infof("Bind to LAN: %v", ifname) err := CheckIpforward(ifname) if err != nil { return err } link, err := netlink.LinkByName(ifname) if err != nil { return err } /// Insert an elem into IfindexParamsMap. ifParams, err := getIfParamsFromLink(link) if err != nil { return err } if err = ifParams.CheckVersionRequirement(c.kernelVersion); err != nil { return err } if err := c.bpf.IfindexParamsMap.Update(uint32(link.Attrs().Index), ifParams, ebpf.UpdateAny); err != nil { return fmt.Errorf("update IfindexIpsMap: %w", err) } // Insert filters. filterIngress := &netlink.BpfFilter{ FilterAttrs: netlink.FilterAttrs{ LinkIndex: link.Attrs().Index, Parent: netlink.HANDLE_MIN_INGRESS, Handle: netlink.MakeHandle(0x2023, 2), Protocol: unix.ETH_P_ALL, // Priority should be behind of WAN's Priority: 2, }, Fd: c.bpf.bpfPrograms.TproxyLanIngress.FD(), Name: consts.AppName + "_lan_ingress", DirectAction: true, } // Remove and add. _ = netlink.FilterDel(filterIngress) if err := netlink.FilterAdd(filterIngress); err != nil { return fmt.Errorf("cannot attach ebpf object to filter ingress: %w", err) } c.deferFuncs = append(c.deferFuncs, func() error { if err := netlink.FilterDel(filterIngress); err != nil { return fmt.Errorf("FilterDel(%v:%v): %w", ifname, filterIngress.Name, err) } return nil }) return nil } func (c *ControlPlaneCore) setupSkPidMonitor() error { /// Set-up SrcPidMapper. /// Attach programs to support pname routing. // Get the first-mounted cgroupv2 path. cgroupPath, err := detectCgroupPath() if err != nil { return err } // Bind cg programs type cgProg struct { Name string Prog *ebpf.Program Attach ebpf.AttachType } cgProgs := []cgProg{ {Prog: c.bpf.TproxyWanCgSockCreate, Attach: ebpf.AttachCGroupInetSockCreate}, {Prog: c.bpf.TproxyWanCgSockRelease, Attach: ebpf.AttachCgroupInetSockRelease}, //{Prog: c.bpf.TproxyWanCgConnect4, Attach: ebpf.AttachCGroupInet4Connect}, //{Prog: c.bpf.TproxyWanCgConnect6, Attach: ebpf.AttachCGroupInet6Connect}, //{Prog: c.bpf.TproxyWanCgSendmsg4, Attach: ebpf.AttachCGroupUDP4Sendmsg}, //{Prog: c.bpf.TproxyWanCgSendmsg6, Attach: ebpf.AttachCGroupUDP6Sendmsg}, } for _, prog := range cgProgs { attached, err := ciliumLink.AttachCgroup(ciliumLink.CgroupOptions{ Path: cgroupPath, Attach: prog.Attach, Program: prog.Prog, }) if err != nil { return fmt.Errorf("AttachTracing: %v: %w", prog.Prog.String(), err) } c.deferFuncs = append(c.deferFuncs, func() error { if err := attached.Close(); err != nil { return fmt.Errorf("inet6Bind.Close(): %w", err) } return nil }) } return nil } func (c *ControlPlaneCore) bindWan(ifname string) error { c.log.Infof("Bind to WAN: %v", ifname) link, err := netlink.LinkByName(ifname) if err != nil { return err } if link.Attrs().Index == consts.LoopbackIfIndex { return fmt.Errorf("cannot bind to loopback interface") } /// Insert an elem into IfindexParamsMap. ifParams, err := getIfParamsFromLink(link) if err != nil { return err } if err = ifParams.CheckVersionRequirement(c.kernelVersion); err != nil { return err } if err := c.bpf.IfindexParamsMap.Update(uint32(link.Attrs().Index), ifParams, ebpf.UpdateAny); err != nil { return fmt.Errorf("update IfindexIpsMap: %w", err) } /// Set-up WAN ingress/egress TC programs. // Insert TC filters filterEgress := &netlink.BpfFilter{ FilterAttrs: netlink.FilterAttrs{ LinkIndex: link.Attrs().Index, Parent: netlink.HANDLE_MIN_EGRESS, Handle: netlink.MakeHandle(0x2023, 1), Protocol: unix.ETH_P_ALL, Priority: 1, }, Fd: c.bpf.bpfPrograms.TproxyWanEgress.FD(), Name: consts.AppName + "_wan_egress", DirectAction: true, } // Remove and add. _ = netlink.FilterDel(filterEgress) if err := netlink.FilterAdd(filterEgress); err != nil { return fmt.Errorf("cannot attach ebpf object to filter egress: %w", err) } c.deferFuncs = append(c.deferFuncs, func() error { if err := netlink.FilterDel(filterEgress); err != nil && !os.IsNotExist(err) { return fmt.Errorf("FilterDel(%v:%v): %w", ifname, filterEgress.Name, err) } return nil }) filterIngress := &netlink.BpfFilter{ FilterAttrs: netlink.FilterAttrs{ LinkIndex: link.Attrs().Index, Parent: netlink.HANDLE_MIN_INGRESS, Handle: netlink.MakeHandle(0x2023, 1), Protocol: unix.ETH_P_ALL, Priority: 1, }, Fd: c.bpf.bpfPrograms.TproxyWanIngress.FD(), Name: consts.AppName + "_wan_ingress", DirectAction: true, } // Remove and add. _ = netlink.FilterDel(filterIngress) if err := netlink.FilterAdd(filterIngress); err != nil { return fmt.Errorf("cannot attach ebpf object to filter ingress: %w", err) } c.deferFuncs = append(c.deferFuncs, func() error { if err := netlink.FilterDel(filterIngress); err != nil { return fmt.Errorf("FilterDel(%v:%v): %w", ifname, filterIngress.Name, err) } return nil }) return nil }