try to fix checksum problem but failed

README.md

@@ -78,10 +78,21 @@ Note that if you bind dae to WAN only, dae only provide network service for loca
 ## TODO
 
 1. Check dns upstream and source loop (whether upstream is also a client of us) and remind the user to add sip rule.
+
 1. Domain routing performance optimization.
+
 1. Handle the case that nodes do not support UDP.
+
 1. Handle the case that nodes do not support IPv6.
-1. L4Checksum problem. Maybe it is hard to solve.
+
+1. L4Checksum problem.
+
+   If the NIC checksumming offload is enabled, the Linux network stack will make a simple checksum a packet when it is sent out from local. When NIC discovers that the source IP of the packet is the local IP of the NIC, it will checksum it complete this checksum.
+
+   But the problem is, after the Linux network stack, before entering the network card, we modify the source IP of this packet, causing the Linux network stack to only make a simple checksum, and the NIC also assumes that this packet is not sent from local, so no further checksum completing.
+
 1. MACv2 extension extraction.
+
 1. Log to userspace.
+
 1. ...

cmd/run.go

@@ -29,7 +29,7 @@ var (
 			// Read config from --config cfgFile.
 			param, err := readConfig(cfgFile)
 			if err != nil {
-				logrus.Fatalln("readConfig: %w", err)
+				logrus.Fatalln("readConfig:", err)
 			}
 
 			log := logger.NewLogger(param.Global.LogLevel, disableTimestamp)

component/control/dns.go

@@ -263,6 +263,11 @@ func (c *ControlPlane) DnsRespHandler(data []byte, validateRushAns bool) (newDat
 		if e == nil && !exist {
 			// Additional record OPT in the request was ensured, and in normal case the resp should also set it.
 			// This DNS packet may be a rush-answer, and we should reject it.
+			c.log.WithFields(logrus.Fields{
+				"ques": q,
+				"addi": FormatDnsRsc(msg.Additionals),
+				"ans":  FormatDnsRsc(msg.Answers),
+			}).Traceln("DNS rush-answer detected")
 			return nil, SuspectedRushAnswerError
 		}
 	}

component/control/kern/tproxy.c

@@ -12,9 +12,9 @@
 #include "headers/vmlinux.h"
 
 // #include <bpf/bpf_core_read.h>
-#include "headers/bpf_probe_read.h"
 #include "headers/bpf_endian.h"
 #include "headers/bpf_helpers.h"
+#include "headers/bpf_probe_read.h"
 
 // #define __DEBUG_ROUTING
 // #define __PRINT_ROUTING_RESULT
@@ -290,8 +290,7 @@ struct {
 
 // Functions:
 
-static __always_inline bool equal_ipv6_format(const __be32 x[4],
-                                              const __be32 y[4]) {
+static __always_inline bool equal16(const __be32 x[4], const __be32 y[4]) {
 #if __clang_major__ >= 10
   return ((__be64 *)x)[0] == ((__be64 *)y)[0] &&
          ((__be64 *)x)[1] == ((__be64 *)y)[1];
@@ -315,12 +314,51 @@ static __always_inline __u32 l4_checksum_off(__u8 proto, __u8 ihl) {
   return ETH_HLEN + ihl * 4 + l4_checksum_rel_off(proto);
 }
 
+static __always_inline int bpf_update_offload_l4cksm_32(struct __sk_buff *skb,
+                                                        __u32 l4_cksm_off,
+                                                        __be32 old,
+                                                        __be32 new) {
+  int ret;
+  __sum16 cksm;
+  if ((ret = bpf_skb_load_bytes(skb, l4_cksm_off, &cksm, sizeof(cksm)))) {
+    return ret;
+  }
+  //  bpf_printk("before: %x", bpf_ntohs(cksm));
+  cksm =
+      bpf_htons(bpf_ntohs(cksm) + bpf_ntohs(*(__be16 *)&new) +
+                bpf_ntohs(*((__be16 *)&new + 1)) - bpf_ntohs(*(__be16 *)&old) -
+                bpf_ntohs(*((__be16 *)&old + 1)));
+  if ((ret = bpf_skb_store_bytes(skb, l4_cksm_off, &cksm, sizeof(cksm), 0))) {
+    return ret;
+  }
+  //  bpf_printk("after: %x", bpf_ntohs(cksm));
+  return 0;
+}
+
+static __always_inline int bpf_update_offload_l4cksm_16(struct __sk_buff *skb,
+                                                        __u32 l4_cksm_off,
+                                                        __be16 old,
+                                                        __be16 new) {
+  int ret;
+  __sum16 cksm;
+  if ((ret = bpf_skb_load_bytes(skb, l4_cksm_off, &cksm, sizeof(cksm)))) {
+    return ret;
+  }
+  //  bpf_printk("before: %x", bpf_ntohs(cksm));
+  cksm = bpf_htons(bpf_ntohs(cksm) + bpf_ntohs(new) - bpf_ntohs(old));
+  if ((ret = bpf_skb_store_bytes(skb, l4_cksm_off, &cksm, sizeof(cksm), 0))) {
+    return ret;
+  }
+  //  bpf_printk("after: %x", bpf_ntohs(cksm));
+  return 0;
+}
+
 static __always_inline int rewrite_ip(struct __sk_buff *skb, __u8 ipversion,
                                       __u8 proto, __u8 ihl, __be32 old_ip[4],
                                       __be32 new_ip[4], bool is_dest,
                                       bool calc_l4_cksm) {
   // Nothing to do.
-  if (equal_ipv6_format(old_ip, new_ip)) {
+  if (equal16(old_ip, new_ip)) {
     return 0;
   }
   // bpf_printk("%pI6->%pI6", old_ip, new_ip);
@@ -350,6 +388,13 @@ static __always_inline int rewrite_ip(struct __sk_buff *skb, __u8 ipversion,
         bpf_printk("bpf_l4_csum_replace: %d", ret);
         return ret;
       }
+    } else {
+      // NIC checksum offload path. But problem remains. FIXME.
+      if ((ret = bpf_update_offload_l4cksm_32(skb, l4_cksm_off, _old_ip,
+                                              _new_ip))) {
+        bpf_printk("bpf_update_offload_cksm_32: %d", ret);
+        return ret;
+      }
     }
     // bpf_skb_load_bytes(skb, l4_cksm_off, &test, sizeof(test));
     // bpf_printk("rewrite ip after: %x", test);
@@ -646,8 +691,8 @@ static __always_inline int adjust_udp_len(struct __sk_buff *skb, __u16 oldlen,
 
   // Calculate checksum and store the new value.
   int ret;
-  if (calc_l4_cksm) {
   __u32 udp_csum_off = l4_checksum_off(IPPROTO_UDP, ihl);
+  if (calc_l4_cksm) {
     // replace twice because len exists both pseudo hdr and hdr.
     if ((ret = bpf_l4_csum_replace(
              skb, udp_csum_off, oldlen, newlen,
@@ -661,6 +706,13 @@ static __always_inline int adjust_udp_len(struct __sk_buff *skb, __u16 oldlen,
       bpf_printk("bpf_l4_csum_replace newudplen: %d", ret);
       return ret;
     }
+  } else {
+    // NIC checksum offload path. But problem remains. FIXME.
+    if ((ret =
+             bpf_update_offload_l4cksm_16(skb, udp_csum_off, oldlen, newlen))) {
+      bpf_printk("bpf_update_offload_cksm: %d", ret);
+      return ret;
+    }
   }
   if ((ret = bpf_skb_store_bytes(
            skb, (__u32)ETH_HLEN + ihl * 4 + offsetof(struct udphdr, len),
@@ -1028,7 +1080,7 @@ routing(const __u32 flag[6], const void *l4_hdr, const __be32 saddr[4],
         good_subrule = true;
       }
     } else if (match_set->type == MatchType_ProcessName) {
-      if (_is_wan && equal_ipv6_format(match_set->pname, _pname)) {
+      if (_is_wan && equal16(match_set->pname, _pname)) {
         good_subrule = true;
       }
     } else if (match_set->type == MatchType_Final) {
@@ -1437,7 +1489,8 @@ int tproxy_lan_egress(struct __sk_buff *skb) {
   struct if_params *ifparams =
       bpf_map_lookup_elem(&ifindex_params_map, &ifindex);
   if (unlikely(!ifparams)) {
-    return -1;
+    bpf_printk("no ifparams found for ifindex %u", ifindex);
+    return TC_ACT_OK;
   }
 
   bool disable_checksum = ipversion == 4 ? ifparams->tx_l4_cksm_ip4_offload
@@ -1450,7 +1503,7 @@ int tproxy_lan_egress(struct __sk_buff *skb) {
   }
   __be32 tproxy_ip[4];
   ret = ip_is_host(ipversion, ifparams, saddr, tproxy_ip);
-  if (!(ret == 1) || !equal_ipv6_format(saddr, tproxy_ip)) {
+  if (!(ret == 1) || !equal16(saddr, tproxy_ip)) {
     return TC_ACT_OK;
   }
 

component/control/udp.go

@@ -89,6 +89,9 @@ func (c *ControlPlane) RelayToUDP(lConn *net.UDPConn, to netip.AddrPort, isDNS b
 			if err != nil {
 				if errors.Is(err, SuspectedRushAnswerError) {
 					// Reject DNS rush-answer.
+					c.log.WithFields(logrus.Fields{
+						"from": from,
+					}).Tracef("DNS rush-answer rejected")
 					return err
 				}
 				c.log.Debugf("DnsRespHandler: %v", err)