Failover Container Install

failover_container_install

#!/usr/bin/env bash


# no prompts
export DEBIAN_FRONTEND=noninteractive

# fix https on apt-get
apt-get install -y apt-transport-https software-properties-common

# Set locale and timezone
locale-gen en_US.UTF-8
echo 'America/New_York' > /etc/timezone
dpkg-reconfigure --frontend noninteractive tzdata

# Install docker
apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
echo 'deb https://apt.dockerproject.org/repo ubuntu-trusty main' | tee -a /etc/apt/sources.list.d/docker.list
apt-get update
apt-get install -y linux-image-extra-$(uname -r)
apt-get install -y docker-engine

# Install haproxy
add-apt-repository -y ppa:vbernat/haproxy-1.6
apt-get update
apt-get install -y haproxy vim-haproxy

# Install utilities
apt-get -q -y install htop vnc4server keepalived vim nmap curl htop tcpdump tcptrack iotop iftop iperf fluxbox xfce4-terminal autossh btrfs-tools mutt ntpdate whois lsof traceroute openvswitch-switch ntp psmisc strace krb5-user samba winbind libnss-winbind libpam-winbind ssh

# Configure docker to use btrfs
echo 'DOCKER_OPTS="-s btrfs"' >> /etc/default/docker
mv /var/lib/docker /var/lib/docker.bak
ln -s /mnt/btrfs/data/docker /var/lib/docker

# Install pipework
git clone https://github.com/jpetazzo/pipework.git
mv pipework/pipework /usr/local/sbin/

# This is samba's configuration file it is being read to the
# $smbconf variable and will be written to a file later. The
# configuration is for the hcpss.org domain and will map ad
# unique id's to local uid/gid's between 10000 and 999999
# User local home directorys will be stored in the /home/
# directory. Passwords are also cached locally in the event
# AD is unavailable.
#/etc/samba/smb.conf
read -r -d '' smbconf <<HEREDOC
[global]
    workgroup = HCPSS
    security = ads
    realm = HCPSS.ORG
    domain master = no
    local master = no
    preferred master = no
    load printers = no
    idmap backend = tdb
    idmap uid = 10000-999999
    idmap gid = 10000-999999
    idmap config HCPSS:backend = rid
    idmap config HCPSS:range = 10000-999999
    winbind enum users = yes
    winbind enum groups = yes
    winbind use default domain = yes
    winbind nested groups = yes
    winbind refresh tickets = yes
    winbind offline logon = true
    template homedir = /home/%U
    template shell = /bin/bash
    client use spnego = yes
    client ntlmv2 auth = yes
    encrypt passwords = yes
    restrict anonymous = 2
    log file = /var/log/samba/samba.log
    log level = 2
HEREDOC

# This is the kerberos config file. It is being stored in
# the $krb5conf variable and will be written to disk later.
# This defines the domains and servers available to the
# system to authenticate and generate kerberos tickets. This
# config is using the entire "hcpss.org" set of hosts to
# authenticate against.
#/etc/krb5.conf
read -r -d '' krb5conf <<HEREDOC
[libdefaults]
  ticket_lifetime = 24h
  default_realm = HCPSS.ORG
  forwardable = true

[realms]
  HCPSS.ORG = {
    kdc = hcpss.org
    default_domain = HCPSS.ORG
  }

[domain_realm]
  .hcpss.org = HCPSS.ORG
  hcpss.org = HCPSS.ORG

[kdc]
  profile = /etc/krb5kdc/kdc.conf

[appdefaults]
  pam = {
    debug = false
    ticket_lifetime = 36000
    renew_lifetime = 36000
    forwardable = true
    krb4_convert = false
  }

[logging]
  kdc = FILE:/var/log/krb5kdc.log
  admin_server = FILE:/var/log/kadmin.log
  default = FILE:/var/log/krb5lib.log
HEREDOC

# This is the config file for nsswitch. It is stored in
# the $nsswitchconf variable and will be written to disk
# later. This adds winbind to the default passwd, group,
# and shadow sources.
#/etc/nsswitch.conf
read -r -d '' nsswitchconf <<HEREDOC
passwd:         compat winbind
group:          compat winbind
shadow:         compat winbind
hosts:          files dns
networks:       files
protocols:      db files
services:       db files
ethers:         db files
rpc:            db files
netgroup:       nis
HEREDOC

# This is the config file for sshd. It is stored in the
# $pamdsshd variable and will be written to disk later.
# this adds the requirement for pam_mkhomedir.so. This
# ensures that the home directory will be created for
# users in the directory that have not logged into the
# system.
#/etc/pam.d/sshd
read -r -d '' pamdsshd <<HEREDOC
@include common-auth
account    required     pam_nologin.so
@include common-account
session [success=ok ignore=ignore module_unknown=ignore default=bad]        pam_selinux.so close
session    optional     pam_mkhomedir.so        skel=/etc/skel/ umask=0022
session    required     pam_loginuid.so
session    optional     pam_keyinit.so force revoke
@include common-session
session    optional     pam_motd.so  motd=/run/motd.dynamic noupdate
session    required     pam_limits.so
session    required     pam_env.so user_readenv=1 envfile=/etc/default/locale
session [success=ok ignore=ignore module_unknown=ignore default=bad]        pam_selinux.so open
@include common-password
HEREDOC

# If there is a config file for kerberos on the
# system rename it so it's no longer used by the
# kerberos/winbind service. Write the config file
# mentioned above to /etc/krb5.conf
if [ -f /etc/krb5.conf ]
then
  mv /etc/krb5.conf /etc/krb5.conf"$(date +%s)"
fi
echo "$krb5conf" > /etc/krb5.conf

# If there is a config file for nsswitch on the
# system rename it so it's no longer used by the
# system's auth service. Write the config file
# mentioned above to /etc/nsswitch.conf
if [ -f /etc/nsswitch.conf ]
then
  mv /etc/nsswitch.conf /etc/nsswitch.conf"$(date +%s)"
fi
echo "$nsswitchconf" > /etc/nsswitch.conf

# If there is a config file for samba on the
# system rename it so it's no longer used by the
# samba service. Write the config file
# mentioned above to /etc/samba/smb.conf
if [ -f /etc/samba/smb.conf ]
then
  mv /etc/samba/smb.conf /etc/samba/smb.conf"$(date +%s)"
fi
echo "$smbconf" > /etc/samba/smb.conf

# If there is a pam config file for sshd on the
# system rename it so it's no longer used by the
# pam service. Write the config file
# mentioned above to /etc/pam.d/sshd
if [ -f /etc/pam.d/sshd ]
then
  mv /etc/pam.d/sshd /etc/pamdsshd"$(date +%s)"
fi
echo "$pamdsshd" > /etc/pam.d/sshd

# Add domain admins to sudoers so anyone who
# is a member of the "domain admins" group in
# AD will sudo capabilities on the system
echo '%domain\ admins ALL=(ALL) ALL' >> /etc/sudoers

# Restrict ssh logins to admins so all hcpss
# users cannot login to the system. Only domain
# admins or members of the local adm or root groups
# will be able to login
echo 'AllowGroups ssh adm root "domain admins"' >> /etc/ssh/sshd_config

# Set home to use btrfs
mv /home /home.bak
ln -s /mnt/btrfs/data/home /home

# btrfs-snapshot rotation script
read -r -d '' btrfssnapshot <<HEREDOC
#!/bin/bash

# Parse arguments:
SOURCE=\$1
TARGET=\$2
SNAP=\$3
COUNT=\$4
QUIET=\$5

# Function to display usage:
usage() {
    scriptname=`/usr/bin/basename \$0`
cat <<EOF
\$scriptname: Take and rotate snapshots on a btrfs file system

  Usage:
  \$scriptname source target snap_name count [-q]

  source:    path to make snaphost of
  target:    snapshot directory
  snap_name: Base name for snapshots, to be appended to
             date "+%F_%H-%M-%S_%Z"
  count:     Number of snapshots in the timestamp-@snap_name format to
             keep at one time for a given snap_name.
  [-q]:      Be quiet.

Example for crontab:
15,30,45  * * * *   root    /usr/local/bin/btrfs-snapshot / /.btrfs quarterly 4 -q
0         * * * *   root    /usr/local/bin/btrfs-snapshot / /.btrfs hourly 8 -q

Example for anacrontab:
1             10      daily_snap      /usr/local/bin/btrfs-snapshot / /.btrfs daily 8
7             30      weekly_snap     /usr/local/bin/btrfs-snapshot / /.btrfs weekly 5
@monthly      90      monthly_snap    /usr/local/bin/btrfs-snapshot / /.btrfs monthly 3


EOF
    exit
}

# Basic argument checks:
if [ -z \$COUNT ] ; then
    echo "COUNT is not provided."
    usage
fi

if [ ! -z \$6 ] ; then
    echo "Too many options."
    usage
fi

if [ -n "\$QUIET" ] && [ "x\$QUIET" != "x-q"  ] ; then
    echo "Option 4 is either -q or empty. Given: \"\$QUIET\""
    usage
fi

# check that source exists
if [ ! -d "\$SOURCE" ]; then
  if [ -z \$QUIET ]; then
      echo "\$SOURCE does not exist or is not mounted"
  fi
  exit 1
fi

# \$max_snap is the highest number of snapshots that will be kept for \$SNAP.
max_snap=\$((\$COUNT -1))

# Clean up older snapshots:
for i in `ls \$TARGET|sort |grep @\${SNAP}|head -n -\${max_snap}`; do
    cmd="/sbin/btrfs subvolume delete \$TARGET/\$i"
    if [ -z \$QUIET ]; then
        echo \$cmd
    fi
    \$cmd >/dev/null
done

# Create new snapshot:
cmd="/sbin/btrfs subvolume snapshot \$SOURCE \$TARGET/`date "+%F_%H-%M-%S_%Z-@\${SNAP}"`"
if [ -z \$QUIET ]; then
    echo \$cmd
fi
\$cmd >/dev/null
HEREDOC

# create btrfs-snapshot rotation script
echo "$btrfssnapshot" > /usr/local/bin/btrfs-snapshot
chmod +x /usr/local/bin/btrfs-snapshot

read -r -d '' crontab <<HEREDOC
# Edit this file to introduce tasks to be run by cron.
#
# Each task to run has to be defined through a single line
# indicating with different fields when the task will be run
# and what command to run for the task
#
# To define the time you can provide concrete values for
# minute (m), hour (h), day of month (dom), month (mon),
# and day of week (dow) or use '*' in these fields (for 'any').#
# Notice that tasks will be started based on the cron's system
# daemon's notion of time and timezones.
#
# Output of the crontab jobs (including errors) is sent through
# email to the user the crontab file belongs to (unless redirected).
#
# For example, you can run a backup of all your user accounts
# at 5 a.m every week with:
# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/
#
# For more information see the manual pages of crontab(5) and cron(8)
#
# m h  dom mon dow   command
*/5 *  *   *   *     /usr/local/bin/btrfs-snapshot /mnt/btrfs/data /mnt/btrfs/.data-snapshots fivemin 12 -q
  0 *  *   *   *     /usr/local/bin/btrfs-snapshot /mnt/btrfs/data /mnt/btrfs/.data-snapshots  hourly 24 -q
  0 3  *   *   *     /usr/local/bin/btrfs-snapshot /mnt/btrfs/data /mnt/btrfs/.data-snapshots   daily 14 -q
HEREDOC

# install cron for btrfs-snapshot
echo "$crontab" > /tmp/crontab.tmp; crontab /tmp/crontab.tmp

# a script to mount the failover device and start docker
read -r -d '' failover <<HEREDOC
#!/usr/bin/env bash

if [ -z "\$1" ]
then
  echo "Please specify a device to mount"
  echo "  failover /dev/drbd1"
  exit 1
fi

device="\$1"
mountpoint="/mnt/btrfs"

# make sure the mount folder exists
if [ ! -d "\$mountpoint" ]
then
  mkdir "\$mountpoint"
fi

#mount the device
if mount "\$device" "\$mountpoint"
then
  mv /var/lib/docker/network/files /tmp/olddockernetworkfiles\$(date +%s)
  service docker start
fi
HEREDOC

# a script to stop docker and unmount the failover device
read -r -d '' failback <<HEREDOC
#!/usr/bin/env bash

mountpoint="/mnt/btrfs"
device="\$(mount | grep "\$mountpoint" | awk '{print \$1}')"

# check device is mounted
if [ -z "\$device" ]
then
  echo "Can't find device, \$mountpoint not mounted"
  exit 1
fi

# stop docker and all services
service docker stop
volume_in_use=1
while [ ! -z \$volume_in_use ]
do
  pids=\$(lsof -n | grep "\$mountpoint" | awk '{print \$2}' | uniq)
  if [ ! -z "\$pids" ]
  then
    echo "Killing \$pids"
    kill \$pids
    sleep 1
  else
    volume_in_use=
  fi
done

# unmount device
umount "\$mountpoint"
HEREDOC

# create the failover and failback scripts
echo "$failover" > /usr/local/sbin/failover
echo "$failback" > /usr/local/sbin/failback
chmod +x /usr/local/sbin/failover /usr/local/sbin/failback

# a script to start all other services
read -r -d '' startup <<HEREDOC
#!/usr/bin/env bash

# check that this is the root user
if [ "\$UID" -ne "0" ]
then
  exit
fi

# start ssh
if ! pgrep sshd>/dev/null; then service ssh start; fi

# start keepalived
if ! pgrep keepalived>/dev/null; then service keepalived start; fi

# start haproxy
if ! pgrep haproxy>/dev/null; then service haproxy start; fi

# start winbind
if ! pgrep winbindd>/dev/null; then service winbind start; fi

# start smb
if ! pgrep smbd>/dev/null; then service smbd start; fi

# start nmb
if ! pgrep nmbd>/dev/null; then service nmbd start; fi

# start openvswitch
if ! pgrep ovs-vswitchd>/dev/null; then service openvswitch-switch start; fi

# start cron
if ! pgrep cron>/dev/null; then
  cron -f &
fi
HEREDOC

# install startup script
echo "$startup" > /usr/local/sbin/startup
chmod +x /usr/local/sbin/startup
# set the startup script to run when the container starts
echo "/usr/local/sbin/startup" >> /etc/bash.bashrc
# start the services
/usr/local/sbin/startup


# script to create a network bridge
read -r -d '' ovsbridge <<HEREDOC
gateway=\$(ip route | grep default | awk '{print \$3}')
ip=\$(ip route get \$gateway | awk '{print \$NF; exit}')
interface=\$(ip route | grep \$ip | awk '{print \$3}')
cidr=\$(ip route | grep \$ip | awk '{print \$1}' | sed 's/^[^\/]\+//g')

# check for ip address
if [ -z "\$ip" ]
then
  echo "No default interface available"
  exit 1
fi

# check that current interface isn't already a bridge
if [ "\$interface" == "ovsbr0" ]
then
  echo "ovsbr0 is already the default interface"
  exit 1
fi

# delete the old bridge if it exists
if ovs-vsctl show | grep ovsbr0 >/dev/null
then
  ovs-vsctl del-br ovsbr0
fi

# create the bridge
ovs-vsctl add-br ovsbr0
ovs-vsctl add-port ovsbr0 \$interface
ip addr flush dev \$interface
ip link set ovsbr0 up
ip addr add "\$ip\$cidr" dev ovsbr0
ip route add default via \$gateway dev ovsbr0
# ping the gateway for upstream arp cache
ping -c 3 \$gateway
HEREDOC

# install the bridge script
echo "$ovsbridge" > /usr/local/sbin/ovsbridge
chmod +x /usr/local/sbin/ovsbridge

#bridge the interface
/usr/local/sbin/ovsbridge