FreeBSD ZFS Jail

jail.conf

interface     = "lo1";
host.hostname = "$name";
ip4.addr      = "172.16.0.$n";
path          = "/jail/run/$name";
exec.prestart = "/sbin/zfs clone zroot/jail/template/10.2-RELEASE/root@10.2-RELEASE-p8 zroot/jail/run/$name &&
                 /usr/sbin/sysrc -f /jail/run/$name/etc/rc.conf hostname=$name &&
                 /usr/sbin/sysrc -f /jail/run/$name/etc/rc.conf sshd_enable=YES &&
                 /usr/sbin/sysrc -f /jail/run/$name/etc/rc.conf sshd_flags=\"-o ListenAddress=172.16.0.$n\" &&
                 /usr/sbin/pw -R /jail/run/$name useradd -n u01 -m -w random &&
                 /usr/sbin/pw -R /jail/run/$name lock root &&
                 /sbin/mount -t devfs -o ruleset=4 . /jail/run/$name/dev";
exec.start    = "/bin/sh /etc/rc";
exec.stop     = "/bin/sh /etc/rc.shutdown";
exec.poststop = "/sbin/umount /jail/run/$name/dev; 
                 /sbin/zfs destroy zroot/jail/run/$name";
exec.clean;

j01 { $n = 1; }
j02 { $n = 2; }
j03 { $n = 3; }
j04 { $n = 4; }
j05 { $n = 5; }
j06 { $n = 6; }
j07 { $n = 7; }

jail.sh

#!/bin/sh 

[ -z "${DEBUG}" ] || set -x

ZPOOL=${ZPOOL:-zroot}
JAIL_ROOT=${JAIL_ROOT:-/jail}
JAIL_RELEASE=${JAIL_RELEASE:-$(sysctl -n kern.osrelease | sed -e 's/-p[0-9]*$//')}
JAIL_ARCH=${JAIL_ARCH:-$(sysctl -n hw.machine_arch)}
JAIL_DIST=${JAIL_ROOT}/dist/${JAIL_RELEASE}
JAIL_TEMPLATE=${JAIL_ROOT}/template/${JAIL_RELEASE}/root
JAIL_RUN=${JAIL_ROOT}/run
JAIL_IP=${JAIL_IP:-172.16.0.%d}
JAIL_NAME=${JAIL_NAME:-j%03d}
DIST_SRC=${DIST_SRC:-http://ftp.freebsd.org/pub/FreeBSD/releases/${JAIL_ARCH}/${JAIL_RELEASE}}

_err() {
    echo $@ >&2
    exit 1
}

_create_zfs_datasets() {
    zfs create -o compression=lz4 -o mountpoint=${JAIL_ROOT} -p ${ZPOOL}/jail
    zfs create -p ${ZPOOL}/jail/dist/${JAIL_RELEASE}
    zfs create -p ${ZPOOL}/jail/template/${JAIL_RELEASE}/root
    zfs create -p ${ZPOOL}/jail/run
}

_download_distfiles() {
    for f in base.txz doc.txz lib32.txz
    do
        fetch -o ${JAIL_DIST}/${f} ${DIST_SRC}/${f}
    done
}

_unpack_template() {
    for f in base.txz doc.txz lib32.txz
    do
        tar -C ${JAIL_TEMPLATE} -xf ${JAIL_DIST}/${f}
    done
}

_freebsd_update_chroot() {

    local chroot=${1}
    if [ -z "${chroot}" ]
    then 
        _err ERROR: Must specify chroot directory
    fi
    
    UNAME_r=${JAIL_RELEASE} PAGER="/usr/bin/tail -n0" freebsd-update -b ${chroot} \
                                                          -d ${chroot}/var/db/freebsd-update/ \
                                                          -f ${chroot}/etc/freebsd-update.conf fetch
    UNAME_r=${JAIL_RELEASE} PAGER=/bin/cat freebsd-update -b ${chroot} \
                                                          -d ${chroot}/var/db/freebsd-update/ \
                                                          -f ${chroot}/etc/freebsd-update.conf install
}

_configure_template(){
    sysrc -f ${JAIL_TEMPLATE}/etc/rc.conf sendmail_enable=NO \
                                          sendmail_submit_enable=NO \
                                          sendmail_outbound_enable=NO \
                                          sendmail_msp_queue_enable=NO \
                                          syslogd_enable=NO 
    /usr/sbin/pw -R ${JAIL_TEMPLATE} usermod -n root -w no
    /usr/sbin/pw -R ${JAIL_TEMPLATE} useradd -n u01 -m -w no

    for f in /etc/resolv.conf /etc/localtime
    do
        cp $f ${JAIL_TEMPLATE}/$f
    done

    chroot ${JAIL_TEMPLATE} env ASSUME_ALWAYS_YES=YES pkg bootstrap 
    chroot ${JAIL_TEMPLATE} env ASSUME_ALWAYS_YES=1 pkg update
}

_snapshot_template() {
    local tag=${1}
    if [ -z "${tag}" ]
    then 
        tag=$(cut -f 3,4 -d '|' ${JAIL_TEMPLATE}/var/db/freebsd-update/tag | sed -e 's/|/-p/')
    fi
    zfs snapshot ${ZPOOL}/jail/template/${JAIL_RELEASE}/root@${tag}
}

_clone_template() {
    if [ "$1" == "--tag" ] 
    then
        local tag=${2}
        if [ -z ${tag} ]
        then
            err USAGE: tag needed
        fi
        shift 2
    else    
        local tag=$(cut -f 3,4 -d '|' ${JAIL_TEMPLATE}/var/db/freebsd-update/tag | sed -e 's/|/-p/')
    fi
    local id=${1:-$(uuidgen)}
    zfs clone ${ZPOOL}/jail/template/${JAIL_RELEASE}/root@${tag} ${ZPOOL}/jail/run/${id}

    ## Local configuration

    /usr/sbin/sysrc -f ${JAIL_RUN}/${id}/etc/rc.conf hostname=${id} >/dev/null

    ## End local configuration

    [ -t 1 ] && printf "Cloned jail: "
    echo $id
}

_find_jail_path() {
    local path=${1}
    if [ ! -z "${path}" -a $(ls ${JAIL_RUN} | grep "^${path}" | wc -l) -eq 1 ] 
    then
        echo ${JAIL_RUN}/$(ls ${JAIL_RUN} | grep "^${path}")
    else
        return 1
    fi
}

_find_jail_id() {
    local path=$(_find_jail_path ${1})
    local id=${path##*/}
    if [ ! -z "${id}" -a $(/usr/sbin/jls host.hostname jid | grep "^${id}" | wc -l) -eq 1 ] 
    then
        /usr/sbin/jls host.hostname jid | grep "^${id}" | cut -d' ' -f2
    else
        return 1
    fi
}

_chroot_jail() {
    local path=$(_find_jail_path ${1})
    if [ -z "${path}" ]
    then 
        _err ERROR: Cant find jail - $1
    fi
    echo "--> ${path}"
    chroot ${path}
}

_run() {

    if [ "$1" == "--tag" ] 
    then
        local tag=${2}
        if [ -z ${tag} ]
        then
            err USAGE: tag needed
        fi
        shift 2
    else    
        local tag=$(cut -f 3,4 -d '|' ${JAIL_TEMPLATE}/var/db/freebsd-update/tag | sed -e 's/|/-p/')
    fi

    if [ $# -ne 1 ]
    then
        _err 'USAGE: _run <n>'
    fi

    local n=${1}
    local ip=$(printf ${JAIL_IP} ${n})
    local name=$(printf ${JAIL_NAME} ${n})
    local path=${JAIL_RUN}/${name}

    if jls -j ${name} >/dev/null 2>&1 
    then
        _err ERROR: Jail ${name} already running
    fi

    if zfs list ${ZPOOL}/jail/run/${name} >/dev/null 2>&1
    then
        _err ERROR: ZFS dataset ${name} exists
    fi

    zfs clone ${ZPOOL}/jail/template/${JAIL_RELEASE}/root@${tag} ${ZPOOL}/jail/run/${name}
    /usr/sbin/sysrc -f ${path}/etc/rc.conf hostname=${name} \
                                           sshd_enable=YES \
                                           sshd_flags="-o ListenAddress=${ip}" >/dev/null
    /usr/sbin/pw -R ${path} lock root
    /usr/sbin/pw -R ${path} usermod -n u01 -w random

    /usr/sbin/jail -c host.hostname=${name} \
                      name=${name} \
                      path=${path} \
                      ip4.addr=${ip} \
                      exec.start="/bin/sh /etc/rc" \
                      exec.clean \
                      mount.devfs >/dev/null || _err ERROR: Unable to start jail

}

_kill() {

    if [ $# -ne 1 ]
    then
        _err 'USAGE: _run <n>'
    fi

    local n=${1}
    local name=$(printf ${JAIL_NAME} ${n})
    local path=${JAIL_RUN}/${name}
    local jid=$(jls -j ${name} jid 2>/dev/null)

    if [ -z "${jid}" ]
    then
        _err ERROR: Jail ${name} not running
    fi

    /usr/sbin/jexec $jid /bin/sh /etc/rc.shutdown
    /bin/pkill -j $jid 
    /bin/sleep 0.5

    if jls -j ${name} >/dev/null 2>&1
    then 
        echo ERROR: Jail still running - removing
        /usr/sbin/jail -r ${jid}
    fi

    umount ${path}/dev
    zfs destroy ${ZPOOL}/jail/run/${name}
}

_start_jail() {

    if [ $# -ne 2 ]
    then
        _err 'USAGE: _start_jail <id> <interface|address>'
    fi

    local path=$(_find_jail_path ${1})
    if [ -z "${path}" ]
    then 
        _err ERROR: Cant find jail - $1
    fi

    local ip=${2}
    if [ -z "${ip}" ]
    then 
        _err 'ERROR: Need to specify ip4 address <interface|address>'
    fi

    # Configure SSHD
    /usr/sbin/sysrc -f ${path}/etc/rc.conf sshd_enable=YES
    /usr/sbin/sysrc -f ${path}/etc/rc.conf sshd_flags="-o ListenAddress=${ip}"
    # Set user pw
    /usr/sbin/pw -R ${path} usermod -n u01 -w random

    /usr/sbin/jail -c host.hostname=${path##*/} \
                      path=${path} \
                      ip4.addr="${ip}" \
                      exec.start="/bin/sh /etc/rc" \
                      exec.clean \
                      mount.devfs || _err ERROR: Unable to start jail

    [ -t 1 ] && printf "Started jail: "
    echo $id
}
    
_stop_jail() {
    local id=$(_find_jail_id ${1})
    local path=$(_find_jail_path ${1})

    if [ -z "${id}" ]
    then 
        _err ERROR: Jail not running - $1
    fi

    /usr/sbin/jexec $id /bin/sh /etc/rc.shutdown
    /bin/pkill -l -j $id 
    /bin/sleep 1

    if _find_jail_id ${1} >/dev/null 2>&1
    then 
        echo ERROR: Jail still running - removing
        /usr/sbin/jail -r ${id}
    fi

    mount | grep -q ${path}/dev && umount ${path}/dev
}

_shell() {
    local id=$(_find_jail_id ${1})
    shift
    if [ -z "${id}" ]
    then 
        _err ERROR: Jail not running - $1
    fi

    if [ -z "$*" ]
    then
        /usr/sbin/jexec $id /bin/sh -
    else
        /usr/sbin/jexec $id $@
    fi
}

_list() {
    /bin/ls -1 ${JAIL_RUN}
}

_running() {
    jls path ip4.addr | sed -ne "s^${JAIL_RUN}/^^p"
}

_usage() {
    cat <<EOM
Usage: $0 [cmd]

    init                    - Initialise ZFS datasets, downlaod/upadte dist 
                              and create template (equivalent to: create/
                              download/unpack/update/configure/snapshot)
    create                  - Initialise ZFS datasets
    download                - Download distfiles 
                              (default same release/arch as host) 
    unpack                  - Unpack distfiles
    update [dir]            - Update distribution in chroot 
                              (default $JAIL_TEMPLATE)
    configure               - Configure template
    snapshot [tag]          - Snapshot template (default is osrelease)
    clone [--tag <tag> [id] - Clone template from <tag> (default latest 
                              osrelease) to create new jail with given 
                              <id> (default UUID)
    find <id>               - Find jail path for <id>
    id <id>                 - Find JID for jail <id> (if running)
    chroot <id>             - Chroot into jail <id> (for local configuration)
    start <id> <if|addr>    - Start jail <id> with specified address
    stop <id>               - Stop jail <id>
    shell <id> [cmd]        - Open shell (or run <cmd>) in jail <id>
    list                    - List jail ids
    running                 - List running jail ids

    <id> is jail id (by default UUID) - can be abbreviated if unique

    System defaults can be configured by env variables (see code)

EOM
}

_main() {

    local cmd="${1:-help}"
    shift || true

    case "$cmd" in
        init)      _create_zfs_datasets
                   _download_distfiles
                   _unpack_template
                   _freebsd_update_chroot ${JAIL_TEMPLATE}
                   _configure_template
                   _snapshot_template
                   ;;
        create)    _create_zfs_datasets
                   ;;
        download)  _download_distfiles
                   ;;
        unpack)    _unpack_template
                   ;;
        update)    _freebsd_update_chroot ${1:-${JAIL_TEMPLATE}}
                   ;;
        configure) _configure_template
                   ;;
        snapshot)  _snapshot_template $@
                   ;;
        clone)     _clone_template $@
                   ;;
        find)      _find_jail_path $@ || _err ERROR: Cant find jail matching $@
                   ;;
        id)        _find_jail_id $@ || _err ERROR: Jail not running
                   ;;
        chroot)    _chroot_jail $@
                   ;;
        start)     _start_jail $@
                   ;;
        stop)      _stop_jail $@
                   ;;
        shell)     _shell $@
                   ;;
        list)      _list $@
                   ;;
        running)   _running $@
                   ;;
        run)       _run $@
                   ;;
        kill)      _kill $@
                   ;;
        *|help)    _usage
                   ;;
        esac
}

_main $@