Setup a docker-libnetwork like vxlan overlay network on two hosts manually

docker_vxlan.sh

#!/usr/bin/env bash

function setup_overlay() {
    ip netns add overlay
    ip netns exec overlay ip li ad dev br0 type bridge
    ip li add dev vxlan42 type vxlan id 42 l2miss l3miss proxy learning dstport 4789
    ip link set vxlan42 netns overlay
    ip netns exec overlay ip li set dev vxlan42 name vxlan1
    ip netns exec overlay ip li set dev vxlan1 master br0
    ip li add dev vetha1 mtu 1450 type veth peer name vetha2 mtu 1450
    ip li set dev vetha1 netns overlay
    ip netns exec overlay ip li set dev vetha1 name veth2
    ip netns exec overlay ip li set dev veth2 master br0
    ip netns exec overlay ip ad add dev br0 $bridge_gatway_cidr
    ip netns exec overlay ip li set vxlan1 up
    ip netns exec overlay ip li set veth2 up
    ip netns exec overlay ip li set br0 up
    ip netns add container
    ip li set dev vetha2 netns container
    ip netns exec container ip li set dev vetha2 name eth1 address $container1_mac_addr
    ip netns exec container ip ad add dev eth1 $container1_ip_cidr
    ip netns exec container ip li set dev eth1 up
    ip netns exec overlay ip neighbor add $container2_ip lladdr $container2_mac_addr dev vxlan1 nud permanent
    ip netns exec overlay bridge fdb add to $container2_mac_addr dst $container2_host_ip dev vxlan1
}

# setup overlay on host1
bridge_gatway_cidr='10.0.0.1/24'
container1_ip_cidr='10.0.0.2/24'
container1_mac_addr='02:42:0a:00:00:02'
container2_ip='10.0.0.3'
container2_mac_addr='02:42:0a:00:00:03'
container2_host_ip='192.168.33.12'
setup_overlay

# setup overlay on host2
bridge_gatway_cidr='10.0.0.1/24'
container1_ip_cidr='10.0.0.3/24'
container1_mac_addr='02:42:0a:00:00:03'
container2_ip='10.0.0.2'
container2_mac_addr='02:42:0a:00:00:02'
container2_host_ip='192.168.33.11'
setup_overlay

# check everything on each host
ip netns exec container ip ad
ip netns exec overlay ip ad
ip netns exec overlay ip neighbor
ip netns exec overlay bridge fdb show dev vxlan1
## successful output
root@docker-1:/home/vagrant# ip netns exec container ip ad
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
9: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 02:42:0a:00:00:02 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.2/24 scope global eth1
       valid_lft forever preferred_lft forever
    inet6 fe80::42:aff:fe00:2/64 scope link 
       valid_lft forever preferred_lft forever
root@docker-1:/home/vagrant# ip netns exec overlay ip ad
1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
2: br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default 
    link/ether 22:4e:05:cf:43:57 brd ff:ff:ff:ff:ff:ff
    inet 10.0.0.1/24 scope global br0
       valid_lft forever preferred_lft forever
    inet6 fe80::204e:5ff:fecf:4357/64 scope link 
       valid_lft forever preferred_lft forever
8: vxlan1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master br0 state UNKNOWN group default 
    link/ether a2:9c:84:6c:3f:49 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::a09c:84ff:fe6c:3f49/64 scope link 
       valid_lft forever preferred_lft forever
10: veth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc pfifo_fast master br0 state UP group default qlen 1000
    link/ether 22:4e:05:cf:43:57 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::204e:5ff:fecf:4357/64 scope link 
       valid_lft forever preferred_lft forever
root@docker-1:/home/vagrant# ip netns exec overlay ip neighbor
10.0.0.3 dev vxlan1 lladdr 02:42:0a:00:00:03 PERMANENT
root@docker-1:/home/vagrant# ip netns exec overlay bridge fdb show dev vxlan1
02:42:0a:00:00:03 vlan 0 master br0 
a2:9c:84:6c:3f:49 vlan 0 master br0 permanent
02:42:0a:00:00:03 dst 192.168.33.12 self permanent

# ping container2 on host1
ip netns exec container ping -c 10 10.0.0.3
## successful output
root@docker-1:/home/vagrant# ip netns exec container ping -c 10 10.0.0.3
PING 10.0.0.3 (10.0.0.3) 56(84) bytes of data.
64 bytes from 10.0.0.3: icmp_seq=1 ttl=64 time=0.879 ms
64 bytes from 10.0.0.3: icmp_seq=2 ttl=64 time=0.558 ms
64 bytes from 10.0.0.3: icmp_seq=3 ttl=64 time=0.576 ms
64 bytes from 10.0.0.3: icmp_seq=4 ttl=64 time=0.614 ms
64 bytes from 10.0.0.3: icmp_seq=5 ttl=64 time=0.521 ms
64 bytes from 10.0.0.3: icmp_seq=6 ttl=64 time=0.389 ms
64 bytes from 10.0.0.3: icmp_seq=7 ttl=64 time=0.551 ms
64 bytes from 10.0.0.3: icmp_seq=8 ttl=64 time=0.565 ms
64 bytes from 10.0.0.3: icmp_seq=9 ttl=64 time=0.488 ms
64 bytes from 10.0.0.3: icmp_seq=10 ttl=64 time=0.531 ms

--- 10.0.0.3 ping statistics ---
10 packets transmitted, 10 received, 0% packet loss, time 9008ms
rtt min/avg/max/mdev = 0.389/0.567/0.879/0.119 ms

## tcpdump sample on host1
root@docker-1:/home/vagrant# tcpdump -vv -n -s 0 -e -i eth1
tcpdump: listening on eth1, link-type EN10MB (Ethernet), capture size 262144 bytes
07:04:11.684802 08:00:27:f8:73:79 > 08:00:27:d9:18:aa, ethertype IPv4 (0x0800), length 148: (tos 0x0, ttl 64, id 35902, offset 0, flags [none], proto UDP (17), length 134)
    192.168.33.11.40825 > 192.168.33.12.4789: [no cksum] VXLAN, flags [I] (0x08), vni 42
02:42:0a:00:00:02 > 02:42:0a:00:00:03, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 49746, offset 0, flags [DF], proto ICMP (1), length 84)
    10.0.0.2 > 10.0.0.3: ICMP echo request, id 1938, seq 1, length 64
07:04:11.685190 08:00:27:d9:18:aa > 08:00:27:f8:73:79, ethertype IPv4 (0x0800), length 148: (tos 0x0, ttl 64, id 30027, offset 0, flags [none], proto UDP (17), length 134)
    192.168.33.12.37490 > 192.168.33.11.4789: [no cksum] VXLAN, flags [I] (0x08), vni 42
02:42:0a:00:00:03 > 02:42:0a:00:00:02, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 64, id 54785, offset 0, flags [none], proto ICMP (1), length 84)
    10.0.0.3 > 10.0.0.2: ICMP echo reply, id 1938, seq 1, length 64

# clean up on each host
ip netns del overlay
ip netns del container

vxlan_module.sh

# default linux vxlan module port
$ cat /sys/module/vxlan/parameters/udp_port 
8472

# 新建vxlan.conf
# cat /etc/modprobe.d/vxlan.conf

#### Set the VXLAN UDP port ####
options vxlan udp_port=4789

# 重新加载vxlan模块
# rmmod vxlan
# modprobe -v vxlan