gistfile1.txt

I have an vpn tunnel which generates this device

22: tun0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1390 qdisc fq_codel state UNKNOWN group default qlen 500

    link/none

    inet 172.19.60.64/32 scope global tun0

       valid_lft forever preferred_lft forever

    inet6 fe80::3d43:4244:cb7e:839c/64 scope link stable-privacy

       valid_lft forever preferred_lft forever

then I setup a default route in a dedicated route table "kwai"

 ❯ ip r l table kwai

default dev tun0 scope link

then I setup an ip rule that lookup kwai when packets have fwmark 0x1

 ❯ ip rule list

0:      from all lookup local

20000:  from all fwmark 0x14 lookup zerotier

20000:  from all fwmark 0x8 lookup direct

20000:  from all fwmark 0x1 lookup kwai      <---------------

30000:  from all lookup cn

32766:  from all lookup main

32767:  from all lookup default

then I setup an iptable mangle rule that mark OUTPUT packets with mark 0x1 when they are in ipset "kwai"

 ❯ sudo iptables-save -c

# Generated by iptables-save v1.8.5 on Fri Jul 10 16:17:15 2020

*mangle

:PREROUTING ACCEPT [5050:1128761]

:INPUT ACCEPT [2426:548833]

:FORWARD ACCEPT [2623:579806]

:OUTPUT ACCEPT [6331:827817]

:POSTROUTING ACCEPT [8954:1407623]

:LIBVIRT_PRT - [0:0]

[141641:34122184] -A PREROUTING -m set --match-set gfwlist dst -j MARK --set-xmark 0x8/0xffffffff

[0:0] -A PREROUTING -m set --match-set us dst -j MARK --set-xmark 0x14/0xffffffff

[0:0] -A PREROUTING -s 10.0.8.2/32 -j MARK --set-xmark 0x14/0xffffffff

[0:0] -A PREROUTING -s 10.0.9.2/32 -j MARK --set-xmark 0x8/0xffffffff

[0:0] -A PREROUTING -s 172.16.238.0/24 -j MARK --set-xmark 0x14/0xffffffff

[0:0] -A PREROUTING -m set --match-set kwai dst -j MARK --set-xmark 0x1/0xffffffff

[651334:92487827] -A OUTPUT -m set --match-set gfwlist dst -j MARK --set-xmark 0x8/0xffffffff

[0:0] -A OUTPUT -m owner --uid-owner 1001 -j MARK --set-xmark 0x8/0xffffffff

[0:0] -A OUTPUT -m set --match-set us dst -j MARK --set-xmark 0x14/0xffffffff

[1176:76376] -A OUTPUT -m set --match-set kwai dst -j MARK --set-xmark 0x1/0xffffffff            <---------------

[3837859:2196528106] -A POSTROUTING -j LIBVIRT_PRT

[55:18040] -A LIBVIRT_PRT -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill

COMMIT

# Completed on Fri Jul 10 16:17:15 2020

# Generated by iptables-save v1.8.5 on Fri Jul 10 16:17:15 2020

*filter

:INPUT ACCEPT [1325265:3084856669]

:FORWARD ACCEPT [0:0]

:OUTPUT ACCEPT [1897574:379432684]

:LIBVIRT_FWI - [0:0]

:LIBVIRT_FWO - [0:0]

:LIBVIRT_FWX - [0:0]

:LIBVIRT_INP - [0:0]

:LIBVIRT_OUT - [0:0]

[532543:367578891] -A INPUT -s 192.168.122.0/24 -j ACCEPT

[1834282:3781281557] -A INPUT -j LIBVIRT_INP

[38083:139308425] -A INPUT -d 127.0.0.1/32 -j ACCEPT

[41486:40864219] -A INPUT -s 172.26.0.0/16 -j ACCEPT

[509:103598] -A INPUT -s 192.168.2.0/24 -j ACCEPT

[64:3320] -A INPUT -p tcp -m tcp --dport 22 -j REJECT --reject-with icmp-port-unreachable

[1:44] -A INPUT -p tcp -m tcp --dport 80 -j REJECT --reject-with icmp-port-unreachable

[0:0] -A INPUT -p tcp -m tcp --dport 8888 -j REJECT --reject-with icmp-port-unreachable

[0:0] -A INPUT -p tcp -m tcp --dport 12345 -j REJECT --reject-with icmp-port-unreachable

[128:12231] -A INPUT -s 192.168.122.0/24 -j ACCEPT

[642737:1445332880] -A FORWARD -d 192.168.122.109/32 -o virbr0 -j ACCEPT

[0:0] -A FORWARD -d 192.168.122.109/32 -o virbr0 -j ACCEPT

[728407:100677228] -A FORWARD -j LIBVIRT_FWX

[728407:100677228] -A FORWARD -j LIBVIRT_FWI

[728407:100677228] -A FORWARD -j LIBVIRT_FWO

[0:0] -A FORWARD -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu

[2466196:650451802] -A OUTPUT -j LIBVIRT_OUT

[0:0] -A OUTPUT -p tcp -m owner --uid-owner 1001 -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu

[0:0] -A LIBVIRT_FWI -d 192.168.122.0/24 -o virbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT

[0:0] -A LIBVIRT_FWI -o virbr0 -j REJECT --reject-with icmp-port-unreachable

[728407:100677228] -A LIBVIRT_FWO -s 192.168.122.0/24 -i virbr0 -j ACCEPT

[0:0] -A LIBVIRT_FWO -i virbr0 -j REJECT --reject-with icmp-port-unreachable

[0:0] -A LIBVIRT_FWX -i virbr0 -o virbr0 -j ACCEPT

[282:19706] -A LIBVIRT_INP -i virbr0 -p udp -m udp --dport 53 -j ACCEPT

[0:0] -A LIBVIRT_INP -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT

[13:4478] -A LIBVIRT_INP -i virbr0 -p udp -m udp --dport 67 -j ACCEPT

[0:0] -A LIBVIRT_INP -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT

[0:0] -A LIBVIRT_OUT -o virbr0 -p udp -m udp --dport 53 -j ACCEPT

[0:0] -A LIBVIRT_OUT -o virbr0 -p tcp -m tcp --dport 53 -j ACCEPT

[55:18040] -A LIBVIRT_OUT -o virbr0 -p udp -m udp --dport 68 -j ACCEPT

[0:0] -A LIBVIRT_OUT -o virbr0 -p tcp -m tcp --dport 68 -j ACCEPT

COMMIT

# Completed on Fri Jul 10 16:17:15 2020

# Generated by iptables-save v1.8.5 on Fri Jul 10 16:17:15 2020

*nat

:PREROUTING ACCEPT [119963:6938268]

:INPUT ACCEPT [25690:1646269]

:OUTPUT ACCEPT [183345:11452980]

:POSTROUTING ACCEPT [129244:8354524]

:LIBVIRT_PRT - [0:0]

[0:0] -A PREROUTING -p udp -m udp --dport 11000 -j DNAT --to-destination 192.168.122.109:11000

[0:0] -A PREROUTING -p tcp -m tcp --dport 11000 -j DNAT --to-destination 192.168.122.109:11000

[0:0] -A PREROUTING -p udp -m udp --dport 3389 -j DNAT --to-destination 192.168.122.109:3389

[3:148] -A PREROUTING -p tcp -m tcp --dport 3389 -j DNAT --to-destination 192.168.122.109:3389

[275386:16495886] -A POSTROUTING -j LIBVIRT_PRT

[299:49107] -A POSTROUTING -s 172.26.0.0/16 -j MASQUERADE

[0:0] -A POSTROUTING -s 172.24.0.0/16 -j MASQUERADE

[0:0] -A POSTROUTING -s 192.168.1.0/24 -j MASQUERADE

[0:0] -A POSTROUTING -s 192.168.12.0/24 -j MASQUERADE

[0:0] -A POSTROUTING -s 192.168.188.0/24 -j MASQUERADE

[0:0] -A POSTROUTING -s 10.0.9.2/32 -j MASQUERADE

[0:0] -A POSTROUTING -s 10.0.8.2/32 -j MASQUERADE

[0:0] -A POSTROUTING -s 10.0.0.2/32 -j MASQUERADE

[35:5356] -A LIBVIRT_PRT -s 192.168.122.0/24 -d 224.0.0.0/24 -j RETURN

[0:0] -A LIBVIRT_PRT -s 192.168.122.0/24 -d 255.255.255.255/32 -j RETURN

[17566:914340] -A LIBVIRT_PRT -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535

[128264:7176867] -A LIBVIRT_PRT -s 192.168.122.0/24 ! -d 192.168.122.0/24 -p udp -j MASQUERADE --to-ports 1024-65535

[9:828] -A LIBVIRT_PRT -s 192.168.122.0/24 ! -d 192.168.122.0/24 -j MASQUERADE

COMMIT

# Completed on Fri Jul 10 16:17:15 2020

Then I setup the ipset "kwai" with ip 10.48.50.8

 ❯ sudo ipset list kwai

Name: kwai

Type: hash:ip

Revision: 4

Header: family inet hashsize 1024 maxelem 655360

Size in memory: 248

References: 2

Number of entries: 1

Members:

10.48.50.8

Then I tried ssh 10.48.50.8, it fails to connect. mtr 10.48.50.8 shows direct route is used instead of the vpn one.

and 

 ❯ ip r g 10.48.50.8

10.48.50.8 via 172.17.27.254 dev wlp3s0 src 172.17.26.175 uid 1000

    cache