DPVS in VirtualBox

README.md

DPVS in VirtualBox

Environment

• DPVS version: v1.7.6, v1.8.4
• Host: Arch Linux
• VirtualBox: 6.1.2
• Virtual machines:
  • centos: as the DPVS load balancer
  • dfly: as the backend real server
• Virtual networks: File -> Preferences -> Network -> Add NAT networks
  • WAN: 172.27.34.0/24, external traffic and VIPs
  • LAN: 192.168.34.0/24, traffic between DPVS and real servers

DPVS VM

• OS: CentOS 7.7
• CPU count: 2
• Memory: 4GB
• Network:
  • Adapter 1:
    • Attached to: NAT Network 172.27.34.0/24
    • Type: Intel PRO/1000 MT Server (e1000)
  • Adapter 1:
    • Attached to: NAT Network 192.168.34.0/24
    • Type: Intel PRO/1000 MT Server (e1000)

Real Server VM

• OS: DragonFly BSD 5.7
• Memory: 1GB
• NIC: NAT Network 192.168.34.0/24, virtio-net

Load Balancer Setup

• Mode: FullNAT
• VIP:
  • 172.27.34.101
  • 172.27.34.102
• LIP:
  • 192.168.34.101
• RIP:
  • 192.168.34.5
• TCP services:
  • 172.27.34.101:80 -> 192.168.34.5:80
  • 172.27.34.102:80 -> 192.168.34.5:81

DPVS Setup

0. Build DPDK
1. Install DPVS
2. Configure DPVS (use this dpvs.conf)
3. Start DPVS by executing start-dpvs.sh
4. Configure FullNAT by executing config-dpvs.sh

Real Server Setup

Run Nginx on ports 80 and 81.

config-dpvs.sh

#!/bin/sh

dpvs_dir="/opt/dpvs"
dpip="${dpvs_dir}/bin/dpip"
ipvsadm="${dpvs_dir}/bin/ipvsadm"

wan_ip="172.27.34.15"
wan_gw="172.27.34.1"
lan_ip="192.168.34.15"
lan_gw="192.168.34.1"

echo "Configuring dpdk0 (addr) ..."
${dpip} addr add ${wan_ip}/24 dev dpdk0
ip addr add ${wan_ip}/24 dev dpdk0.kni
ip link set dpdk0.kni up
sleep 1

echo "Configuring dpdk0 (route) ..."
${dpip} route add default via ${wan_gw} dev dpdk0 metric 100
ip route add default via ${wan_gw} dev dpdk0.kni metric 100
sleep 1

echo "Configuring dpdk1 (addr) ..."
${dpip} addr add ${lan_ip}/24 dev dpdk1
ip addr add ${lan_ip}/24 dev dpdk1.kni
ip link set dpdk1.kni up
sleep 1

echo "Configuring dpdk1 (route) ..."
${dpip} route add default via ${lan_gw} dev dpdk1 metric 200
ip route add default via ${lan_gw} dev dpdk1.kni metric 200
sleep 1

vip1="172.27.34.101"
vip2="172.27.34.102"
lip="192.168.34.101"
rip="192.168.34.5"

echo "Configuring VIP ..."
for vip in ${vip1} ${vip2}; do
	${dpip} addr add ${vip}/32 dev dpdk0
	ip addr add ${vip}/32 dev dpdk0.kni
	${ipvsadm} --add-service --tcp-service ${vip}:80 --scheduler rr
done

echo "Configuring real servers ..."
${ipvsadm} --add-server --tcp-service ${vip1}:80 --real-server ${rip}:80 --fullnat
${ipvsadm} --add-server --tcp-service ${vip2}:80 --real-server ${rip}:81 --fullnat

echo "Configuring local IP for FullNAT ..."
#${dpip} addr add ${lip}/32 dev dpdk1 sapool
${ipvsadm} --add-laddr --tcp-service ${vip1}:80 -z ${lip} -F dpdk1
${ipvsadm} --add-laddr --tcp-service ${vip2}:80 -z ${lip} -F dpdk1

echo "=== DPVS Configured ==="
echo
echo "[[[ dpip addr show ]]]"
${dpip} addr show

echo
echo "[[[ dpip route show ]]]"
${dpip} route show

echo
echo "[[[ ipvsadm --list --numeric ]]]"
${ipvsadm} --list --numeric

echo
echo "[[[ ipvsadm --get-laddr ]]]"
${ipvsadm} --get-laddr

dpvs.conf

!
! DPVS configuration tailored for a VM
! Two NICs and recommended 4GB RAM
!

! global config
global_defs {
    log_level   DEBUG 
    log_file    /var/log/dpvs.log
    ! log_async_mode    on
}

! netif config
netif_defs {
    <init> pktpool_size     8191
    <init> pktpool_cache    256

    <init> device dpdk0 {
        rx {
            queue_number        1
            descriptor_number   256
        }
        tx {
            queue_number        1
            descriptor_number   256
        }
        ! promisc_mode
        kni_name                dpdk0.kni
    }

    <init> device dpdk1 {
        rx {
            queue_number        1
            descriptor_number   256
            rss                 all
        }
        tx {
            queue_number        1
            descriptor_number   256
        }
        ! promisc_mode
        kni_name                dpdk1.kni
    }
}

! worker config (lcores)
worker_defs {
    <init> worker cpu0 {
        type    master
        cpu_id  0
    }

    <init> worker cpu1 {
        type    slave
        cpu_id  1
        port    dpdk0 {
            rx_queue_ids     0
            tx_queue_ids     0
        }
        port    dpdk1 {
            rx_queue_ids     0
            tx_queue_ids     0
        }
    }
}

! timer config
timer_defs {
    # cpu job loops to schedule dpdk timer management
    schedule_interval    500
}

! dpvs neighbor config
neigh_defs {
    <init> unres_queue_length  128
    <init> timeout             60
}

! dpvs ipv4 config
ipv4_defs {
    forwarding                 off
    <init> default_ttl         64
    fragment {
        <init> bucket_number   4096
        <init> bucket_entries  16
        <init> max_entries     4096
        <init> ttl             1
    }
}

! dpvs ipv6 config
ipv6_defs {
    disable                     off
    forwarding                  off
    route6 {
        <init> method           hlist
        recycle_time            10
    }
}

! control plane config
ctrl_defs {
    lcore_msg {
        <init> ring_size                1024
        sync_msg_timeout_us             20000
        priority_level                  low
    }
    ipc_msg {
        <init> unix_domain /var/run/dpvs_ctrl
    }
}

! ipvs config
ipvs_defs {
    conn {
        <init> conn_pool_size       65536
        <init> conn_pool_cache      128
        conn_init_timeout           3
    }

    tcp {
        ! defence_tcp_drop
        timeout {
            none        2
            established 90
            syn_sent    3
            syn_recv    30
            fin_wait    7
            time_wait   7
            close       3
            close_wait  7
            last_ack    7
            listen      120
            synack      30
            last        2
        }
        synproxy {
            synack_options {
                mss             1452
                ttl             63
                sack
                ! wscale
                ! timestamp
            }
            ! defer_rs_syn
            rs_syn_max_retry    3
            ack_storm_thresh    10
            max_ack_saved       3
            conn_reuse_state {
                close
                time_wait
                ! fin_wait
                ! close_wait
                ! last_ack
           }
        }
    }
}

! sa_pool config
sa_pool {
    pool_hash_size   8
}

start-dpvs.sh

#!/bin/sh

dpvs_dir="/opt/dpvs"
dpvs="${dpvs_dir}/bin/dpvs"
dpdk_devbind="${dpvs_dir}/bin/dpdk-devbind.py"

echo "Stopping NetworkManager ..."
systemctl stop NetworkManager

cat dpvs.conf > /etc/dpvs.conf

echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages

echo "Inserting kernel modules ..."
rte_kni_params=""
if modinfo ${dpvs_dir}/kmod/rte_kni.ko | grep -qw carrier; then
	rte_kni_params="carrier=on"
	echo "Inserting rte_kni.ko with parameters: ${rte_kni_params}"
fi
modprobe uio
insmod ${dpvs_dir}/kmod/igb_uio.ko
insmod ${dpvs_dir}/kmod/rte_kni.ko ${rte_kni_params}

${dpdk_devbind} --status-dev net

echo "Binding NICs ..."
NICS=$(${dpdk_devbind} --status-dev net |
	sed -n '/kernel driver/,/^Other/ p' |
	grep 'drv=' | sed "s/'.*'//")
echo "${NICS}" | while read busid if_ drv unused _; do
	ifname="${if_#if=}"
	drv_inuse="${drv#drv=}"
	drv_unused="${unused#unused=}"
	echo "NIC: busid=${busid}, ifname=${ifname}, drv_inuse=${drv_inuse}, drv_unused=${drv_unused}"
	ip link set ${ifname} down
	${dpdk_devbind} -u ${busid}
	sleep 1
	${dpdk_devbind} -b ${drv_unused} ${busid}
	sleep 1
done

${dpdk_devbind} --status-dev net

${dpvs} &
echo "pid(dpvs): $!"

stop-dpvs.sh

#!/bin/sh

dpvs_dir="/opt/dpvs"
dpdk_devbind="${dpvs_dir}/bin/dpdk-devbind.py"

echo "Stopping DPVS ..."
kill $(cat /var/run/dpvs.pid)

echo "Unbinding NICs ..."
NICS=$(${dpdk_devbind} --status-dev net |
	sed -n '/DPDK-compatible/,/^Network/ p' |
	grep 'drv=' | sed "s/'.*'//")
echo "${NICS}" | while read busid drv unused; do
	drv_inuse="${drv#drv=}"
	drv_unused="${unused#unused=}"
	echo "NIC: busid=${busid}, drv_inuse=${drv_inuse}, drv_unused=${drv_unused}"
	${dpdk_devbind} -u ${busid}
	sleep 1
	${dpdk_devbind} -b ${drv_unused} ${busid}
	sleep 1
done

${dpdk_devbind} --status-dev net

rmmod rte_kni
rmmod igb_uio
rmmod uio

echo "Starting NetworkManager ..."
systemctl start NetworkManager