chore/docs: support to unroll route loop and update troubleshooting.md (#158)

control/kern/tproxy.c

@@ -19,6 +19,7 @@
 // #define __PRINT_ROUTING_RESULT
 // #define __PRINT_SETUP_PROCESS_CONNNECTION
 // #define __REMOVE_BPF_PRINTK
+// #define __UNROLL_ROUTE_LOOP
 
 // #define likely(x) x
 // #define unlikely(x) x
@@ -137,7 +138,7 @@ struct routing_result {
   __u32 pid;
 };
 
-struct dst_routing_result {
+struct __attribute__((packed)) dst_routing_result {
   __be32 ip[4];
   __be16 port;
   __u16 recognize;
@@ -1005,7 +1006,9 @@ route(const __u32 flag[6], const void *l4hdr, const __be32 saddr[4],
 
   // Unroll can give less instructions but more memory consumption when loading.
   // We disable it here to support more poor memory devices.
-  // #pragma unroll
+#ifdef __UNROLL_ROUTE_LOOP
+#pragma unroll
+#endif
   for (__u32 i = 0; i < MAX_MATCH_SET_LEN; i++) {
     __u32 k = i; // Clone to pass code checker.
     match_set = bpf_map_lookup_elem(&routing_map, &k);
@@ -1016,8 +1019,9 @@ route(const __u32 flag[6], const void *l4hdr, const __be32 saddr[4],
 #ifdef __DEBUG_ROUTING
       key = match_set->type;
       bpf_printk("key(match_set->type): %llu", key);
-      bpf_printk("Skip to judge. bad_rule: %d, good_subrule: %d", isdns_must_goodsubrule_badrule&0b10,
+      bpf_printk("Skip to judge. bad_rule: %d, good_subrule: %d",
-                 isdns_must_goodsubrule_badrule&0b1);
+                 isdns_must_goodsubrule_badrule & 0b10,
+                 isdns_must_goodsubrule_badrule & 0b1);
 #endif
       goto before_next_loop;
     }
@@ -1103,7 +1107,9 @@ route(const __u32 flag[6], const void *l4hdr, const __be32 saddr[4],
 
   before_next_loop:
 #ifdef __DEBUG_ROUTING
-    bpf_printk("good_subrule: %d, bad_rule: %d", isdns_must_goodsubrule_badrule&0b10, isdns_must_goodsubrule_badrule&0b1);
+    bpf_printk("good_subrule: %d, bad_rule: %d",
+               isdns_must_goodsubrule_badrule & 0b10,
+               isdns_must_goodsubrule_badrule & 0b1);
 #endif
     if (match_set->outbound != OUTBOUND_LOGICAL_OR) {
       // This match_set reaches the end of subrule.
@@ -1119,7 +1125,7 @@ route(const __u32 flag[6], const void *l4hdr, const __be32 saddr[4],
       isdns_must_goodsubrule_badrule &= ~0b10;
     }
 #ifdef __DEBUG_ROUTING
-    bpf_printk("_bad_rule: %d", isdns_must_goodsubrule_badrule&0b1);
+    bpf_printk("_bad_rule: %d", isdns_must_goodsubrule_badrule & 0b1);
 #endif
     if ((match_set->outbound & OUTBOUND_LOGICAL_MASK) !=
         OUTBOUND_LOGICAL_MASK) {
@@ -2065,7 +2071,7 @@ static int __always_inline _update_map_elem_by_cookie(const __u64 cookie) {
   We extract "sddm-helper" from it.
   */
   unsigned long loc, j, last_slash = -1;
-  #pragma unroll
+#pragma unroll
   for (loc = 0, j = 0; j < MAX_ARG_LEN_TO_PROBE;
        ++j, loc = ((loc + 1) & (MAX_ARG_SCANNER_BUFFER_SIZE - 1))) {
     // volatile unsigned long k = j; // Cheat to unroll.

docs/getting-started/troubleshooting.md

@@ -10,6 +10,12 @@ Solution:
 
 Compile dae with CFLAG `-D__REMOVE_BPF_PRINTK`. See [build-by-yourself](build-by-yourself.md).
 
+## No network after `dae suspend`
+
+Do not set dae as the DNS in DHCP setting. For example, you can set `223.5.5.5` as DNS in your DHCP setting.
+
+Because dae will not hijack any DNS request if it was suspended.
+
 ## PVE related
 
 - [PVE NIC Hardware passthrough](https://github.com/daeuniverse/dae/issues/43)
@@ -61,3 +67,14 @@ netstat -ulpen|grep 53
 ```
 
 If does, stop the service process or change its listening port from 53 to others. Do not forget to modify `/etc/resolv.conf` to make DNS accessible (for example, with content `nameserver 223.5.5.5`, but do not use `nameserver 127.0.0.1`).
+
+## Failed to load eBPF objects
+
+> FATA[0022] load eBPF objects: field TproxyWanEgress: program tproxy_wan_egress: load program: argument list too long: 1617: (bf) r2 = r6: 1618: (85) call bpf_map_loo (truncated, 992 line(s) omitted)
+
+If you use `clang-13` to compile dae, you may encounter this problem.
+
+There are ways to resolve it:
+
+1. Method 1: Use `clang-15` or higher versions to compile dae. Or just download dae from [releases](https://github.com/daeuniverse/dae/releases).
+2. Method 2: Add CFLAGS `-D__UNROLL_ROUTE_LOOP` while compiling. However, it will increse memory occupation (or swap space) at the eBPF loading stage (about 180MB). For example, compile dae to ARM64 using `make CGO_ENABLE=0 GOARCH=arm64 CFLAGS="-D__UNROLL_ROUTE_LOOP -D__REMOVE_BPF_PRINTK"`.

hack/test/insert.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 lan=docker0
-wan=wlp5s0
+wan=ens192
 
 sudo tc qdisc add dev $lan clsact > /dev/null 2>&1
 sudo tc qdisc add dev $wan clsact > /dev/null 2>&1
@@ -9,8 +9,8 @@ set -ex
 
 sudo rm -rf /sys/fs/bpf/tc/globals/*
 
-# clang -fno-stack-protector -O2 -g -emit-llvm -c control/kern/tproxy.c -o - | llc -march=bpf -mcpu=v3 -mattr=+alu32 -filetype=obj -o foo.o
+# clang -fno-stack-protector -O2 -g -emit-llvm -c ../../control/kern/tproxy.c -o - | llc -march=bpf -mcpu=v3 -mattr=+alu32 -filetype=obj -o foo.o
-clang -O2 -g -Wall -Werror -c control/kern/tproxy.c -target bpf -D__TARGET_ARCH_x86 -o foo.o
+clang -O2 -g -Wall -Werror -c ../../control/kern/tproxy.c -target bpf -D__TARGET_ARCH_x86 -o foo.o
 sudo tc filter del dev $lan ingress
 sudo tc filter del dev $lan egress
 sudo tc filter del dev $wan ingress
@@ -26,4 +26,4 @@ sudo tc filter del dev $lan egress
 sudo tc filter del dev $wan ingress
 sudo tc filter del dev $wan egress
 
-exit 0
+exit 0