opte-util.sh

#!/bin/bash
set -eux

if [[ "$EUID" -ne 0 ]]; then
    echo "Run as root"
    exit
fi

# Physical link over which to simulate the Chelsio links
PHYSICAL_LINK="$(dladm show-phys -p -o LINK | head -1)"

# MAC address of the gateway in the local network
PHYS_GATEWAY_MAC="$(arp -an | grep -w $(netstat -rn  | awk '{ if ($1 == "default") print $2; }') | awk '{ print $4; }')"

# Simulated Chelsio links
VNIC_NAMES=("ul0" "ul1")

function success (
    set +x;
    echo -e "\e[1;36m$1\e[0m"
)
function warn (
    set +x;
    echo -e "\e[1;31m$1\e[0m"
)
function fail {
    warn "$1"
    exit 1
}

# Return the name of a VNIC link if it exists, or the empty string if not.
#
# Arguments:
#   $1: The name of the VNIC to look for
function get_vnic_name_if_exists {
    dladm show-vnic -p -o LINK "$1" 2> /dev/null || echo ""
}

# Create vNICs to represent the Chelsio physical links
function ensure_simulated_chelsios {
    for VNIC in "${VNIC_NAMES[@]}"; do
        if [[ -z "$(get_vnic_name_if_exists "$VNIC")" ]]; then
            dladm create-vnic -t -l "$PHYSICAL_LINK" "$VNIC"
            success "vNIC $VNIC created"
        else
            success "VNIC $VNIC exists"
        fi

        # Configure (v6) IP interfaces atop the vNICs
        ipadm create-addr -t -T addrconf $VNIC/v6
    done
}

# Remove the given interface
#
# Arguments:
#   $1: The name of the vNIC to delete
function try_remove_interface {
    local IFACE="$1"
    if [[ "$(ipadm show-if -p -o IFNAME "$IFACE" 2>/dev/null)" ]]; then
        ipadm delete-if "$IFACE" || warn "Failed to delete interface $IFACE"
    fi
    success "Verified IP interface $IFACE does not exist"
}

# Remove the given vNIC
#
# Arguments:
#   $1: The name of the vNIC to delete
function try_remove_vnic {
    local LINK="$1"
    if [[ "$(get_vnic_name_if_exists "$LINK")" ]]; then
        dladm delete-vnic "$LINK" || warn "Failed to delete vNIC link $LINK"
    fi
    success "Verified vNIC link $LINK does not exist"
}

# Unload the xde driver
function unload_xde_driver {
    local ID="$(modinfo | grep xde | cut -d ' ' -f 1)"
    if [[ "$ID" ]]; then
        modunload -i "$ID" || fail "Failed to unload xde driver"
    fi
    success "Verified the xde kernel driver is unloaded"
}

################################################################################

# Create the vNICs and setup OPTE
function setup {
    # Create simulated links to use with OPTE
    ensure_simulated_chelsios

    # HACK: enable the "external IP hack"
    # Disables encap and turns SNAT into plain NAT
    # Replace w/ boundary services
    sed -i 's/ext_ip_hack = 0/ext_ip_hack = 1/' /kernel/drv/xde.conf
    update_drv xde

    # Configure OPTE to use the simulated Chelsio links
    /opt/oxide/opte/bin/opteadm set-xde-underlay "${VNIC_NAMES[@]}"
    success "OPTE configured to use the simulated Chelsio links"
}

# Clean up the vNICs and OPTE
function cleanup {
    for PORT in $(/opt/oxide/opte/bin/opteadm list-ports | awk 'NR!=1' | awk '{ print $1; }'); do
        for VNIC in $(dladm show-vnic -p -o LINK -l $PORT); do
            try_remove_vnic "$VNIC"
        done
        /opt/oxide/opte/bin/opteadm delete-xde "$PORT"
    done
    unload_xde_driver

    # Undo the "external IP hack"
    sed -i 's/ext_ip_hack = 1/ext_ip_hack = 0/' /kernel/drv/xde.conf

    # Remove the vNICs
    for VNIC in "${VNIC_NAMES[@]}"; do
        try_remove_interface "$VNIC" && try_remove_vnic "$VNIC"
    done

    # Work around bug: clear stale info from ipmgmtd
    svcadm restart ip-interface-management
}

# Create an OPTE port and vNIC for a VM
#
# Arguments:
#   $1: The guest "number"; used to generate addresses
#   $2: The name of the VM
#
# Note: Guests will be assigned external IPs like 192.168.1.2X where
#       X is the guest number.
function add_guest {
    local guest_num="$1"
    local name="$2"
    local gen="0"

    local port="vm_$name$gen"
    local vnic="vnic_$name$gen"

    local mac="A8:40:25:FF:0$gen:0$guest_num"
    local ip="10.0.0.1$guest_num"

    local phys_gateway_mac="$PHYS_GATEWAY_MAC"

    # The gateway MAC & IP address that OPTE exposes to guest NICs
    local gateway_mac="A8:40:25:00:00:01"
    local gateway_ip="10.0.0.1"

    local boundary_services_addr="fd00:99::1"
    local boundary_services_vni=99

    local vpc_vni=10
    local vpc_subnet="10.0.0.0/24"

    local node_num=1
    local source_underlay_addr="fd00:$node_num::1"

    local snat_start="$guest_num"000
    local snat_end="$guest_num"999
    local snat_ip="192.168.1.2$guest_num"
    local external_ip="$snat_ip"

    # Create OPTE port
    /opt/oxide/opte/bin/opteadm create-xde \
        "$port" \
        --private-mac "$mac" \
        --private-ip "$ip" \
        --phys-gw-mac "$phys_gateway_mac" \
        --gateway-mac "$gateway_mac" \
        --gateway-ip "$gateway_ip" \
        --bsvc-addr "$boundary_services_addr" \
        --bsvc-vni "$boundary_services_vni" \
        --vpc-vni "$vpc_vni" \
        --vpc-subnet "$vpc_subnet" \
        --src-underlay-addr "$source_underlay_addr" \
        --snat-start $snat_start \
        --snat-end $snat_end \
        --snat-ip $snat_ip \
        --external-ipv4 $external_ip

    # Create vNIC atop OPTE port
    dladm create-vnic -t -l "$port" -m "$mac" "$vnic"

    # Add secondary MAC for OPTE proxy ARP onto one of the simulated Chelsio links
    dladm set-linkprop -t -p secondary-macs=$mac "${VNIC_NAMES[0]}"

    # Route VM-VM traffic
    /opt/oxide/opte/bin/opteadm add-router-entry \
        -p "$port" \
        "$vpc_subnet" \
        "sub4=$vpc_subnet"

    # Add default gateway
    /opt/oxide/opte/bin/opteadm add-router-entry \
        -p "$port" \
        "0.0.0.0/0" \
        ig
}

case "${1-}" in
    "setup" )
        setup
        ;;

    "cleanup" )
        cleanup
        ;;

    "add-guest" )
        if [[ $# -lt 3 ]]; then
            echo "Usage: $0 add-guest <guest_num> <name>"
            exit 1
        fi

        shift
        add_guest "$@"
        ;;

    *)
        echo "Usage: $0 {setup|cleanup|add-guest}"
        exit 1
        ;;
esac