-
-
Save wido/51cb9880d86f08f73766634d7f6df3f4 to your computer and use it in GitHub Desktop.
#!/usr/bin/env bash | |
# | |
# Use BGP+EVPN for VXLAN with CloudStack instead of Multicast | |
# | |
# Place this file on all KVM hypervisors at /usr/share/modifyvxlan.sh | |
# | |
# More information about BGP and EVPN with FRR: https://vincent.bernat.ch/en/blog/2017-vxlan-bgp-evpn | |
# | |
DSTPORT=4789 | |
# We bind our VXLAN tunnel IP(v4) on Loopback device 'lo' | |
DEV="lo" | |
usage() { | |
echo "Usage: $0: -o <op>(add | delete) -v <vxlan id> -p <pif> -b <bridge name> (-6)" | |
} | |
localAddr() { | |
local FAMILY=$1 | |
if [[ -z "$FAMILY" || $FAMILY == "inet" ]]; then | |
ip -4 -o addr show scope global dev ${DEV} | awk 'NR==1 {gsub("/[0-9]+", "") ; print $4}' | |
fi | |
if [[ "$FAMILY" == "inet6" ]]; then | |
ip -6 -o addr show scope global dev ${DEV} | awk 'NR==1 {gsub("/[0-9]+", "") ; print $4}' | |
fi | |
} | |
addVxlan() { | |
local VNI=$1 | |
local PIF=$2 | |
local VXLAN_BR=$3 | |
local FAMILY=$4 | |
local VXLAN_DEV=vxlan${VNI} | |
local ADDR=$(localAddr ${FAMILY}) | |
echo "local addr for VNI ${VNI} is ${ADDR}" | |
if [[ ! -d /sys/class/net/${VXLAN_DEV} ]]; then | |
ip -f ${FAMILY} link add ${VXLAN_DEV} type vxlan id ${VNI} local ${ADDR} dstport ${DSTPORT} nolearning | |
ip link set ${VXLAN_DEV} up | |
sysctl -qw net.ipv6.conf.${VXLAN_DEV}.disable_ipv6=1 | |
fi | |
if [[ ! -d /sys/class/net/$VXLAN_BR ]]; then | |
ip link add name ${VXLAN_BR} type bridge | |
ip link set ${VXLAN_BR} up | |
sysctl -qw net.ipv6.conf.${VXLAN_BR}.disable_ipv6=1 | |
fi | |
bridge link show|grep ${VXLAN_BR}|awk '{print $2}'|grep "^${VXLAN_DEV}\$" > /dev/null | |
if [[ $? -gt 0 ]]; then | |
ip link set ${VXLAN_DEV} master ${VXLAN_BR} | |
fi | |
} | |
deleteVxlan() { | |
local VNI=$1 | |
local PIF=$2 | |
local VXLAN_BR=$3 | |
local FAMILY=$4 | |
local VXLAN_DEV=vxlan${VNI} | |
ip link set ${VXLAN_DEV} nomaster | |
ip link delete ${VXLAN_DEV} | |
ip link set ${VXLAN_BR} down | |
ip link delete ${VXLAN_BR} type bridge | |
} | |
OP= | |
VNI= | |
FAMILY=inet | |
option=$@ | |
while getopts 'o:v:p:b:6' OPTION | |
do | |
case $OPTION in | |
o) oflag=1 | |
OP="$OPTARG" | |
;; | |
v) vflag=1 | |
VNI="$OPTARG" | |
;; | |
p) pflag=1 | |
PIF="$OPTARG" | |
;; | |
b) bflag=1 | |
BRNAME="$OPTARG" | |
;; | |
6) | |
FAMILY=inet6 | |
;; | |
?) usage | |
exit 2 | |
;; | |
esac | |
done | |
if [[ "$oflag$vflag$pflag$bflag" != "1111" ]]; then | |
usage | |
exit 2 | |
fi | |
lsmod|grep ^vxlan >& /dev/null | |
if [[ $? -gt 0 ]]; then | |
modprobe=`modprobe vxlan 2>&1` | |
if [[ $? -gt 0 ]]; then | |
echo "Failed to load vxlan kernel module: $modprobe" | |
exit 1 | |
fi | |
fi | |
# | |
# Add a lockfile to prevent this script from running twice on the same host | |
# this can cause a race condition | |
# | |
LOCKFILE=/var/run/cloud/vxlan.lock | |
( | |
flock -x -w 10 200 || exit 1 | |
if [[ "$OP" == "add" ]]; then | |
addVxlan ${VNI} ${PIF} ${BRNAME} ${FAMILY} | |
if [[ $? -gt 0 ]]; then | |
exit 1 | |
fi | |
elif [[ "$OP" == "delete" ]]; then | |
deleteVxlan ${VNI} ${PIF} ${BRNAME} ${FAMILY} | |
fi | |
) 200>${LOCKFILE} |
@wido may I know is the script still works for ACS 4.19? Just to be straight, I don't know in detail of how VXLAN and EVPN works and now trying to implement in in my POC environment.
This is my FRR configuration on the host so far.
ip forwarding
ipv6 forwarding
interface ens3f0np0
no ipv6 nd suppress-ra
exit
interface ens3f1np1
no ipv6 nd suppress-ra
exit
router bgp 4200100005
bgp router-id 10.0.118.1
no bgp ebgp-requires-policy
neighbor uplink peer-group
neighbor uplink remote-as external
neighbor ens3f0np0 interface peer-group uplink
neighbor ens3f1np1 interface peer-group uplink
address-family ipv4 unicast
network 10.0.118.1/32
exit-address-family
address-family ipv6 unicast
network 2407:e6c0:0:1::1/128
neighbor uplink activate
neighbor uplink soft-reconfiguration inbound
exit-address-family
address-family l2vpn evpn
neighbor uplink activate
neighbor uplink attribute-unchanged next-hop
advertise-all-vni
exit-address-family
I can see that I'm getting the EVPN routes from my leaf switches (but still don't understand what's RD RT etc).
I tried to use the script to create a VXLAN interface on cloudbr0
for the management network. Am I missing something?
[root@cmpt1 ~]# ./modifyvxlan.sh -o add -v 10027 -b cloudbr0
Usage: ./modifyvxlan.sh: -o <op>(add | delete) -v <vxlan id> -p <pif> -b <bridge name> (-6)
[root@cmpt1 ~]#
Thanks for the help :)
@wido may I know is the script still works for ACS 4.19? Just to be straight, I don't know in detail of how VXLAN and EVPN works and now trying to implement in in my POC environment.
This is my FRR configuration on the host so far.
ip forwarding ipv6 forwarding interface ens3f0np0 no ipv6 nd suppress-ra exit interface ens3f1np1 no ipv6 nd suppress-ra exit router bgp 4200100005 bgp router-id 10.0.118.1 no bgp ebgp-requires-policy neighbor uplink peer-group neighbor uplink remote-as external neighbor ens3f0np0 interface peer-group uplink neighbor ens3f1np1 interface peer-group uplink address-family ipv4 unicast network 10.0.118.1/32 exit-address-family address-family ipv6 unicast network 2407:e6c0:0:1::1/128 neighbor uplink activate neighbor uplink soft-reconfiguration inbound exit-address-family address-family l2vpn evpn neighbor uplink activate neighbor uplink attribute-unchanged next-hop advertise-all-vni exit-address-family
I can see that I'm getting the EVPN routes from my leaf switches (but still don't understand what's RD RT etc).
I tried to use the script to create a VXLAN interface on
cloudbr0
for the management network. Am I missing something?[root@cmpt1 ~]# ./modifyvxlan.sh -o add -v 10027 -b cloudbr0 Usage: ./modifyvxlan.sh: -o <op>(add | delete) -v <vxlan id> -p <pif> -b <bridge name> (-6) [root@cmpt1 ~]#
Thanks for the help :)
Yes, this script should still work with 4.19, no problem at all.
Could you maybe ask this question on the CloudStack mailinglist? I can get back to it there. You can cc me in the e-mail :-)
@wido I've posted a question on the mailing list. Thanks for checking it out later when you're free :)
I cc'ed you in there also.
Hypervisor you mean? Compute node. That is logical. See the script. It takes the IP which is on the loopback interface and is hardcoded to the VXLAN device.
Use unique IPs per host and do not try to change.