BGP+EVPN+VXLAN with Apache CloudStack

modifyvxlan.sh

#!/usr/bin/env bash
#
# Use BGP+EVPN for VXLAN with CloudStack instead of Multicast
#
# Place this file on all KVM hypervisors at /usr/share/modifyvxlan.sh
#
# More information about BGP and EVPN with FRR: https://vincent.bernat.ch/en/blog/2017-vxlan-bgp-evpn
#

DSTPORT=4789

# We bind our VXLAN tunnel IP(v4) on Loopback device 'lo'
DEV="lo"

usage() {
    echo "Usage: $0: -o <op>(add | delete) -v <vxlan id> -p <pif> -b <bridge name> (-6)"
}

localAddr() {
    local FAMILY=$1

    if [[ -z "$FAMILY" || $FAMILY == "inet" ]]; then
       ip -4 -o addr show scope global dev ${DEV} | awk 'NR==1 {gsub("/[0-9]+", "") ; print $4}'
    fi

    if [[ "$FAMILY" == "inet6" ]]; then
       ip -6 -o addr show scope global dev ${DEV} | awk 'NR==1 {gsub("/[0-9]+", "") ; print $4}'
    fi
}

addVxlan() {
    local VNI=$1
    local PIF=$2
    local VXLAN_BR=$3
    local FAMILY=$4
    local VXLAN_DEV=vxlan${VNI}
    local ADDR=$(localAddr ${FAMILY})

    echo "local addr for VNI ${VNI} is ${ADDR}"

    if [[ ! -d /sys/class/net/${VXLAN_DEV} ]]; then
        ip -f ${FAMILY} link add ${VXLAN_DEV} type vxlan id ${VNI} local ${ADDR} dstport ${DSTPORT} nolearning
        ip link set ${VXLAN_DEV} up
        sysctl -qw net.ipv6.conf.${VXLAN_DEV}.disable_ipv6=1
    fi

    if [[ ! -d /sys/class/net/$VXLAN_BR ]]; then
        ip link add name ${VXLAN_BR} type bridge
        ip link set ${VXLAN_BR} up
        sysctl -qw net.ipv6.conf.${VXLAN_BR}.disable_ipv6=1
    fi

    bridge link show|grep ${VXLAN_BR}|awk '{print $2}'|grep "^${VXLAN_DEV}\$" > /dev/null
    if [[ $? -gt 0 ]]; then
        ip link set ${VXLAN_DEV} master ${VXLAN_BR}
    fi
}

deleteVxlan() {
    local VNI=$1
    local PIF=$2
    local VXLAN_BR=$3
    local FAMILY=$4
    local VXLAN_DEV=vxlan${VNI}

    ip link set ${VXLAN_DEV} nomaster
    ip link delete ${VXLAN_DEV}

    ip link set ${VXLAN_BR} down
    ip link delete ${VXLAN_BR} type bridge
}

OP=
VNI=
FAMILY=inet
option=$@

while getopts 'o:v:p:b:6' OPTION
do
  case $OPTION in
  o)    oflag=1
        OP="$OPTARG"
        ;;
  v)    vflag=1
        VNI="$OPTARG"
        ;;
  p)    pflag=1
        PIF="$OPTARG"
        ;;
  b)    bflag=1
        BRNAME="$OPTARG"
        ;;
  6)
        FAMILY=inet6
        ;;
  ?)    usage
        exit 2
        ;;
  esac
done

if [[ "$oflag$vflag$pflag$bflag" != "1111" ]]; then
    usage
    exit 2
fi

lsmod|grep ^vxlan >& /dev/null
if [[ $? -gt 0 ]]; then
    modprobe=`modprobe vxlan 2>&1`
    if [[ $? -gt 0 ]]; then
        echo "Failed to load vxlan kernel module: $modprobe"
        exit 1
    fi
fi


#
# Add a lockfile to prevent this script from running twice on the same host
# this can cause a race condition
#

LOCKFILE=/var/run/cloud/vxlan.lock

(
    flock -x -w 10 200 || exit 1
    if [[ "$OP" == "add" ]]; then
        addVxlan ${VNI} ${PIF} ${BRNAME} ${FAMILY}

        if [[ $? -gt 0 ]]; then
            exit 1
        fi
    elif [[ "$OP" == "delete" ]]; then
        deleteVxlan ${VNI} ${PIF} ${BRNAME} ${FAMILY}
    fi
) 200>${LOCKFILE}

@wido may I know is the script still works for ACS 4.19? Just to be straight, I don't know in detail of how VXLAN and EVPN works and now trying to implement in in my POC environment.

This is my FRR configuration on the host so far.

ip forwarding
ipv6 forwarding

interface ens3f0np0
    no ipv6 nd suppress-ra
exit

interface ens3f1np1
    no ipv6 nd suppress-ra
exit

router bgp 4200100005
    bgp router-id 10.0.118.1
    no bgp ebgp-requires-policy
    neighbor uplink peer-group
    neighbor uplink remote-as external
    neighbor ens3f0np0 interface peer-group uplink
    neighbor ens3f1np1 interface peer-group uplink
    address-family ipv4 unicast
        network 10.0.118.1/32
    exit-address-family
    address-family ipv6 unicast
        network 2407:e6c0:0:1::1/128
        neighbor uplink activate
        neighbor uplink soft-reconfiguration inbound
    exit-address-family
    address-family l2vpn evpn
        neighbor uplink activate
        neighbor uplink attribute-unchanged next-hop
        advertise-all-vni
    exit-address-family

I can see that I'm getting the EVPN routes from my leaf switches (but still don't understand what's RD RT etc).

I tried to use the script to create a VXLAN interface on cloudbr0 for the management network. Am I missing something?

[root@cmpt1 ~]# ./modifyvxlan.sh -o add -v 10027 -b cloudbr0
Usage: ./modifyvxlan.sh: -o <op>(add | delete) -v <vxlan id> -p <pif> -b <bridge name> (-6)
[root@cmpt1 ~]#

Thanks for the help :)

Yes, this script should still work with 4.19, no problem at all.

Could you maybe ask this question on the CloudStack mailinglist? I can get back to it there. You can cc me in the e-mail :-)

@wido I've posted a question on the mailing list. Thanks for checking it out later when you're free :)

I cc'ed you in there also.