feat: support tcp_check_http_method (#77)

common/utils.go

@@ -457,3 +457,12 @@ nextLink:
 	}
 	return Deduplicate(defaultIfs), nil
 }
+
+func IsValidHttpMethod(method string) bool {
+	switch method {
+	case "GET", "POST", "PUT", "PATCH", "DELETE", "COPY", "HEAD", "OPTIONS", "LINK", "UNLINK", "PURGE", "LOCK", "UNLOCK", "PROPFIND", "CONNECT", "TRACE":
+		return true
+	default:
+		return false
+	}
+}

component/outbound/dialer/connectivity_check.go

@@ -118,9 +118,13 @@ func parseIp46FromList(ip []string) *netutils.Ip46 {
 type TcpCheckOption struct {
 	Url *netutils.URL
 	*netutils.Ip46
+	Method string
 }
 
-func ParseTcpCheckOption(ctx context.Context, rawURL []string) (opt *TcpCheckOption, err error) {
+func ParseTcpCheckOption(ctx context.Context, rawURL []string, method string) (opt *TcpCheckOption, err error) {
+	if method == "" {
+		method = http.MethodGet
+	}
 	systemDns, err := netutils.SystemDns()
 	if err != nil {
 		return nil, err
@@ -148,8 +152,9 @@ func ParseTcpCheckOption(ctx context.Context, rawURL []string) (opt *TcpCheckOpt
 		}
 	}
 	return &TcpCheckOption{
-		Url:  &netutils.URL{URL: u},
+		Url:    &netutils.URL{URL: u},
-		Ip46: ip46,
+		Ip46:   ip46,
+		Method: method,
 	}, nil
 }
 
@@ -199,10 +204,11 @@ func ParseCheckDnsOption(ctx context.Context, dnsHostPort []string) (opt *CheckD
 }
 
 type TcpCheckOptionRaw struct {
-	opt *TcpCheckOption
+	opt    *TcpCheckOption
-	mu  sync.Mutex
+	mu     sync.Mutex
-	Log *logrus.Logger
+	Log    *logrus.Logger
-	Raw []string
+	Raw    []string
+	Method string
 }
 
 func (c *TcpCheckOptionRaw) Option() (opt *TcpCheckOption, err error) {
@@ -212,7 +218,7 @@ func (c *TcpCheckOptionRaw) Option() (opt *TcpCheckOption, err error) {
 		ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second)
 		defer cancel()
 		ctx = context.WithValue(ctx, "logger", c.Log)
-		tcpCheckOption, err := ParseTcpCheckOption(ctx, c.Raw)
+		tcpCheckOption, err := ParseTcpCheckOption(ctx, c.Raw, c.Method)
 		if err != nil {
 			return nil, fmt.Errorf("failed to parse tcp_check_url: %w", err)
 		}
@@ -279,7 +285,7 @@ func (d *Dialer) aliveBackground() {
 				}).Debugln("Skip check due to no DNS record.")
 				return false, nil
 			}
-			return d.HttpCheck(ctx, opt.Url, opt.Ip4)
+			return d.HttpCheck(ctx, opt.Url, opt.Ip4, opt.Method)
 		},
 	}
 	tcp6CheckOpt := &CheckOption{
@@ -301,7 +307,7 @@ func (d *Dialer) aliveBackground() {
 				}).Debugln("Skip check due to no DNS record.")
 				return false, nil
 			}
-			return d.HttpCheck(ctx, opt.Url, opt.Ip6)
+			return d.HttpCheck(ctx, opt.Url, opt.Ip6, opt.Method)
 		},
 	}
 	tcp4CheckDnsOpt := &CheckOption{
@@ -529,8 +535,11 @@ func (d *Dialer) Check(timeout time.Duration,
 	return ok, err
 }
 
-func (d *Dialer) HttpCheck(ctx context.Context, u *netutils.URL, ip netip.Addr) (ok bool, err error) {
+func (d *Dialer) HttpCheck(ctx context.Context, u *netutils.URL, ip netip.Addr, method string) (ok bool, err error) {
 	// HTTP(S) check.
+	if method == "" {
+		method = http.MethodGet
+	}
 	cd := &netproxy.ContextDialer{Dialer: d.Dialer}
 	cli := http.Client{
 		Transport: &http.Transport{
@@ -548,7 +557,7 @@ func (d *Dialer) HttpCheck(ctx context.Context, u *netutils.URL, ip netip.Addr)
 			},
 		},
 	}
-	req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
+	req, err := http.NewRequestWithContext(ctx, method, u.String(), nil)
 	if err != nil {
 		return false, err
 	}

component/sniffing/http.go

@@ -8,6 +8,7 @@ package sniffing
 import (
 	"bufio"
 	"bytes"
+	"github.com/daeuniverse/dae/common"
 	"strings"
 	"unicode"
 )
@@ -27,9 +28,7 @@ func (s *Sniffer) SniffHttp() (d string, err error) {
 	if !found {
 		return "", NotApplicableError
 	}
-	switch string(method) {
+	if !common.IsValidHttpMethod(string(method)) {
-	case "GET", "POST", "PUT", "PATCH", "DELETE", "COPY", "HEAD", "OPTIONS", "LINK", "UNLINK", "PURGE", "LOCK", "UNLOCK", "PROPFIND":
-	default:
 		return "", NotApplicableError
 	}
 

component/sniffing/sniffing_bench_test.go

@@ -7,6 +7,7 @@ package sniffing
 
 import (
 	"fmt"
+	"github.com/daeuniverse/dae/common"
 	"testing"
 
 	"github.com/mzz2017/softwind/pkg/fastrand"
@@ -17,7 +18,7 @@ var (
 )
 
 func init() {
-	httpMethods := []string{"GET", "POST", "PUT", "PATCH", "DELETE", "COPY", "HEAD", "OPTIONS", "LINK", "UNLINK", "PURGE", "LOCK", "UNLOCK", "PROPFIND"}
+	httpMethods := []string{"GET", "POST", "PUT", "PATCH", "DELETE", "COPY", "HEAD", "OPTIONS", "LINK", "UNLINK", "PURGE", "LOCK", "UNLOCK", "PROPFIND", "CONNECT", "TRACE"}
 	httpMethodSet = make(map[string]struct{})
 	for _, method := range httpMethods {
 		httpMethodSet[method] = struct{}{}
@@ -39,9 +40,7 @@ func BenchmarkStringSwitch(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		var test [5]byte
 		fastrand.Read(test[:])
-		switch string(test[:]) {
+		if !common.IsValidHttpMethod(string(test[:])) {
-		case "GET", "POST", "PUT", "PATCH", "DELETE", "COPY", "HEAD", "OPTIONS", "LINK", "UNLINK", "PURGE", "LOCK", "UNLOCK", "PROPFIND":
-		default:
 			fmt.Sprintf("%v", string(test[:]))
 		}
 	}

config/config.go

@@ -19,6 +19,7 @@ type Global struct {
 	// We use DirectTcpCheckUrl to check (tcp)*(ipv4/ipv6) connectivity for direct.
 	//DirectTcpCheckUrl string `mapstructure:"direct_tcp_check_url" default:"http://www.qualcomm.cn/generate_204"`
 	TcpCheckUrl               []string      `mapstructure:"tcp_check_url" default:"http://cp.cloudflare.com,1.1.1.1,2606:4700:4700::1111"`
+	TcpCheckHttpMethod        string        `mapstructure:"tcp_check_http_method" default:"CONNECT"` // Use 'CONNECT' because some server implementations bypass accounting for this kind of traffic.
 	UdpCheckDns               []string      `mapstructure:"udp_check_dns" default:"dns.google.com:53,8.8.8.8,2001:4860:4860::8888"`
 	CheckInterval             time.Duration `mapstructure:"check_interval" default:"30s"`
 	CheckTolerance            time.Duration `mapstructure:"check_tolerance" default:"0"`

config/desc.go

@@ -35,15 +35,16 @@ var SectionDescription = map[string]Desc{
 }
 
 var GlobalDesc = Desc{
-	"tproxy_port":     "tproxy port to listen on. It is NOT a HTTP/SOCKS port, and is just used by eBPF program.\nIn normal case, you do not need to use it.",
+	"tproxy_port":           "tproxy port to listen on. It is NOT a HTTP/SOCKS port, and is just used by eBPF program.\nIn normal case, you do not need to use it.",
-	"log_level":       "Log level: error, warn, info, debug, trace.",
+	"log_level":             "Log level: error, warn, info, debug, trace.",
-	"tcp_check_url":   "Node connectivity check.\nHost of URL should have both IPv4 and IPv6 if you have double stack in local.\nConsidering traffic consumption, it is recommended to choose a site with anycast IP and less response.",
+	"tcp_check_url":         "Node connectivity check.\nHost of URL should have both IPv4 and IPv6 if you have double stack in local.\nConsidering traffic consumption, it is recommended to choose a site with anycast IP and less response.",
-	"udp_check_dns":   "This DNS will be used to check UDP connectivity of nodes. And if dns_upstream below contains tcp, it also be used to check TCP DNS connectivity of nodes.\nThis DNS should have both IPv4 and IPv6 if you have double stack in local.",
+	"tcp_check_http_method": "The HTTP request method to `tcp_check_url`. Use 'CONNECT' by default because some server implementations bypass accounting for this kind of traffic.",
-	"check_interval":  "Interval of connectivity check for TCP and UDP",
+	"udp_check_dns":         "This DNS will be used to check UDP connectivity of nodes. And if dns_upstream below contains tcp, it also be used to check TCP DNS connectivity of nodes.\nThis DNS should have both IPv4 and IPv6 if you have double stack in local.",
-	"check_tolerance": "Group will switch node only when new_latency <= old_latency - tolerance.",
+	"check_interval":        "Interval of connectivity check for TCP and UDP",
-	"lan_interface":   "The LAN interface to bind. Use it if you want to proxy LAN.",
+	"check_tolerance":       "Group will switch node only when new_latency <= old_latency - tolerance.",
-	"wan_interface":   "The WAN interface to bind. Use it if you want to proxy localhost. Use \"auto\" to auto detect.",
+	"lan_interface":         "The LAN interface to bind. Use it if you want to proxy LAN.",
-	"allow_insecure":  "Allow insecure TLS certificates. It is not recommended to turn it on unless you have to.",
+	"wan_interface":         "The WAN interface to bind. Use it if you want to proxy localhost. Use \"auto\" to auto detect.",
+	"allow_insecure":        "Allow insecure TLS certificates. It is not recommended to turn it on unless you have to.",
 	"dial_mode": `Optional values of dial_mode are:
 1. "ip". Dial proxy using the IP from DNS directly. This allows your ipv4, ipv6 to choose the optimal path respectively, and makes the IP version requested by the application meet expectations. For example, if you use curl -4 ip.sb, you will request IPv4 via proxy and get a IPv4 echo. And curl -6 ip.sb will request IPv6. This may solve some wierd full-cone problem if your are be your node support that.Sniffing will be disabled in this mode.
 2. "domain". Dial proxy using the domain from sniffing. This will relieve DNS pollution problem to a great extent if have impure DNS environment. Generally, this mode brings faster proxy response time because proxy will re-resolve the domain in remote, thus get better IP result to connect. This policy does not impact routing. That is to say, domain rewrite will be after traffic split of routing and dae will not re-route it.

config/patch.go

@@ -6,6 +6,8 @@
 package config
 
 import (
+	"github.com/daeuniverse/dae/common"
+	"github.com/sirupsen/logrus"
 	"strings"
 
 	"github.com/daeuniverse/dae/common/consts"
@@ -15,10 +17,19 @@ import (
 type patch func(params *Config) error
 
 var patches = []patch{
+	patchTcpCheckHttpMethod,
 	patchEmptyDns,
 	patchMustOutbound,
 }
 
+func patchTcpCheckHttpMethod(params *Config) error {
+	if !common.IsValidHttpMethod(params.Global.TcpCheckHttpMethod) {
+		logrus.Warnf("Unknown HTTP Method '%v'. Fallback to 'CONNECT'.", params.Global.TcpCheckHttpMethod)
+		params.Global.TcpCheckHttpMethod = "CONNECT"
+	}
+	return nil
+}
+
 func patchEmptyDns(params *Config) error {
 	if params.Dns.Routing.Request.Fallback == nil {
 		params.Dns.Routing.Request.Fallback = consts.DnsRequestOutboundIndex_AsIs.String()

control/control_plane.go

@@ -227,7 +227,7 @@ func NewControlPlane(
 	}
 	option := &dialer.GlobalOption{
 		Log:               log,
-		TcpCheckOptionRaw: dialer.TcpCheckOptionRaw{Raw: global.TcpCheckUrl, Log: log},
+		TcpCheckOptionRaw: dialer.TcpCheckOptionRaw{Raw: global.TcpCheckUrl, Log: log, Method: global.TcpCheckHttpMethod},
 		CheckDnsOptionRaw: dialer.CheckDnsOptionRaw{Raw: global.UdpCheckDns},
 		CheckInterval:     global.CheckInterval,
 		CheckTolerance:    global.CheckTolerance,

example.dae

@@ -13,6 +13,10 @@ global {
     #tcp_check_url: 'http://cp.cloudflare.com'
     tcp_check_url: 'http://cp.cloudflare.com,1.1.1.1,2606:4700:4700::1111'
 
+    # The HTTP request method to `tcp_check_url`. Use 'CONNECT' by default because some server implementations bypass
+    # accounting for this kind of traffic.
+    tcp_check_http_method: CONNECT
+
     # This DNS will be used to check UDP connectivity of nodes. And if dns_upstream below contains tcp, it also be used to check
     # TCP DNS connectivity of nodes.
     # First is URL, others are IP addresses if given.