chore: send real ready after listening and clean up thoroughly

cmd/run.go

@@ -68,8 +68,12 @@ func Run(log *logrus.Logger, conf *config.Config) (err error) {
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGKILL, syscall.SIGILL, syscall.SIGUSR1)
 	go func() {
-		sdnotify.Ready()
+		readyChan := make(chan bool, 1)
-		if listener, err = c.ListenAndServe(conf.Global.TproxyPort); err != nil {
+		go func() {
+			<-readyChan
+			sdnotify.Ready()
+		}()
+		if listener, err = c.ListenAndServe(readyChan, conf.Global.TproxyPort); err != nil {
 			log.Errorln("ListenAndServe:", err)
 		}
 		sigs <- nil
@@ -82,13 +86,16 @@ loop:
 			if reloading {
 				reloading = false
 				log.Warnln("[Reload] Serve")
-				sdnotify.Ready()
+				readyChan := make(chan bool, 1)
 				go func() {
-					if err := c.Serve(listener); err != nil {
+					if err := c.Serve(readyChan, listener); err != nil {
 						log.Errorln("ListenAndServe:", err)
 					}
 					sigs <- nil
 				}()
+				<-readyChan
+				sdnotify.Ready()
+				log.Warnln("[Reload] Finished")
 			} else {
 				break loop
 			}

control/control_plane.go

@@ -475,8 +475,13 @@ func (l *Listener) Close() error {
 	return err
 }
 
-func (c *ControlPlane) Serve(listener *Listener) (err error) {
+func (c *ControlPlane) Serve(readyChan chan<- bool, listener *Listener) (err error) {
-
+	sentReady := false
+	defer func() {
+		if !sentReady {
+			readyChan <- false
+		}
+	}()
 	udpConn := listener.packetConn.(*net.UDPConn)
 	/// Serve.
 	// TCP socket.
@@ -511,6 +516,8 @@ func (c *ControlPlane) Serve(listener *Listener) (err error) {
 		cancel()
 		return nil
 	})
+	sentReady = true
+	readyChan <- true
 	go func() {
 		defer cancel()
 		for {
@@ -586,7 +593,7 @@ func (c *ControlPlane) Serve(listener *Listener) (err error) {
 	return nil
 }
 
-func (c *ControlPlane) ListenAndServe(port uint16) (listener *Listener, err error) {
+func (c *ControlPlane) ListenAndServe(readyChan chan<- bool, port uint16) (listener *Listener, err error) {
 	// Listen.
 	var listenConfig = net.ListenConfig{
 		Control: func(network, address string, c syscall.RawConn) error {
@@ -615,7 +622,7 @@ func (c *ControlPlane) ListenAndServe(port uint16) (listener *Listener, err erro
 	}()
 
 	// Serve
-	if err = c.Serve(listener); err != nil {
+	if err = c.Serve(readyChan, listener); err != nil {
 		return nil, fmt.Errorf("failed to serve: %w", err)
 	}
 

control/control_plane_core.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"github.com/cilium/ebpf"
 	ciliumLink "github.com/cilium/ebpf/link"
+	"github.com/mohae/deepcopy"
 	"github.com/safchain/ethtool"
 	"github.com/sirupsen/logrus"
 	"github.com/v2rayA/dae/common"
@@ -23,6 +24,7 @@ import (
 	"regexp"
 )
 
+// coreFlip should be 0 or 1
 var coreFlip = 0
 
 type controlPlaneCore struct {
@@ -33,7 +35,8 @@ type controlPlaneCore struct {
 
 	kernelVersion *internal.Version
 
-	flip int
+	flip     int
+	isReload bool
 }
 
 func newControlPlaneCore(log *logrus.Logger,
@@ -43,7 +46,7 @@ func newControlPlaneCore(log *logrus.Logger,
 	isReload bool,
 ) *controlPlaneCore {
 	if isReload {
-		coreFlip = ((coreFlip & 1) ^ 1) & 1
+		coreFlip = coreFlip&1 ^ 1
 	}
 	return &controlPlaneCore{
 		log:             log,
@@ -52,6 +55,7 @@ func newControlPlaneCore(log *logrus.Logger,
 		outboundId2Name: outboundId2Name,
 		kernelVersion:   kernelVersion,
 		flip:            coreFlip,
+		isReload:        isReload,
 	}
 }
 
@@ -293,6 +297,12 @@ func (c *controlPlaneCore) bindLan(ifname string) error {
 	}
 	// Remove and add.
 	_ = netlink.FilterDel(filterIngress)
+	if !c.isReload {
+		// Clean up thoroughly.
+		filterIngressFlipped := deepcopy.Copy(filterIngress).(*netlink.BpfFilter)
+		filterIngressFlipped.FilterAttrs.Handle ^= 1
+		_ = netlink.FilterDel(filterIngressFlipped)
+	}
 	if err := netlink.FilterAdd(filterIngress); err != nil {
 		return fmt.Errorf("cannot attach ebpf object to filter ingress: %w", err)
 	}
@@ -319,6 +329,12 @@ func (c *controlPlaneCore) bindLan(ifname string) error {
 	}
 	// Remove and add.
 	_ = netlink.FilterDel(filterEgress)
+	if !c.isReload {
+		// Clean up thoroughly.
+		filterEgressFlipped := deepcopy.Copy(filterEgress).(*netlink.BpfFilter)
+		filterEgressFlipped.FilterAttrs.Handle ^= 1
+		_ = netlink.FilterDel(filterEgressFlipped)
+	}
 	if err := netlink.FilterAdd(filterEgress); err != nil {
 		return fmt.Errorf("cannot attach ebpf object to filter egress: %w", err)
 	}
@@ -406,8 +422,14 @@ func (c *controlPlaneCore) bindWan(ifname string) error {
 		Name:         consts.AppName + "_wan_egress",
 		DirectAction: true,
 	}
-	// Remove and add.
 	_ = netlink.FilterDel(filterEgress)
+	// Remove and add.
+	if !c.isReload {
+		// Clean up thoroughly.
+		filterEgressFlipped := deepcopy.Copy(filterEgress).(*netlink.BpfFilter)
+		filterEgressFlipped.FilterAttrs.Handle ^= 1
+		_ = netlink.FilterDel(filterEgressFlipped)
+	}
 	if err := netlink.FilterAdd(filterEgress); err != nil {
 		return fmt.Errorf("cannot attach ebpf object to filter egress: %w", err)
 	}
@@ -430,8 +452,14 @@ func (c *controlPlaneCore) bindWan(ifname string) error {
 		Name:         consts.AppName + "_wan_ingress",
 		DirectAction: true,
 	}
-	// Remove and add.
 	_ = netlink.FilterDel(filterIngress)
+	// Remove and add.
+	if !c.isReload {
+		// Clean up thoroughly.
+		filterIngressFlipped := deepcopy.Copy(filterIngress).(*netlink.BpfFilter)
+		filterIngressFlipped.FilterAttrs.Handle ^= 1
+		_ = netlink.FilterDel(filterIngressFlipped)
+	}
 	if err := netlink.FilterAdd(filterIngress); err != nil {
 		return fmt.Errorf("cannot attach ebpf object to filter ingress: %w", err)
 	}