Notes for setting up python-beaver as a log shipper to a RPC logstash server

install-python-beaver.sh

###
### On the Logstash Server
###

# Setup the tcp listener
cat >/etc/logstash/conf.d/01-input.conf <<EOL
#===============================================================================
input {
    syslog {
        port => 5544
        codec => json
        tags => [ "syslog-port" ]
    }
    tcp {
        port => 5140
        codec => json
        tags => [ "tcp-port" ]
    }
}
#===============================================================================
EOL
 
# Restart Logstash
service logstash restart
 
###
### On the Logstash Client
###

# Set the client and server addresses
CLIENT_ADDRESS=$(ip addr show dev eth1 | awk '/inet / {print $2}' | cut -d/ -f1)
LOGSTASH_ADDRESS='172.29.236.207'

# Install python-beaver
pip install -i https://pypi.python.org/simple beaver==33.3.0

# Prepare the configuration files
mkdir -p /etc/beaver/conf.d
cat >/etc/beaver/beaver.conf <<EOL
[beaver]
logstash_version: 1
hostname: ${CLIENT_ADDRESS}
transport: tcp
tcp_host: ${LOGSTASH_ADDRESS}
tcp_port: 5140
EOL

cat >/etc/beaver/conf.d/mysql <<EOL
[/var/log/mysql/mysql-slow.log]
multiline_regex_after: (^#\sTime:\s)
multiline_regex_before: (.*)
tags: mysql,msql-slow
add_field: @fields[program],mysql,@source_host,aio1_galera_container-0fb1a31b
EOL

cat >/etc/beaver/conf.d/syslog <<EOL
[/var/log/syslog]
tags: syslog
add_field: @source_host,aio1_galera_container-0fb1a31b
EOL

# Execute the beaver log shipper
beaver -c /etc/beaver/beaver.conf

###
### Contents of /etc/init.d/beaver
###

#!/bin/bash -
### BEGIN INIT INFO
# Provides:          beaver
# Required-Start:    $local_fs $remote_fs $network
# Required-Stop:     $local_fs $remote_fs $network
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: Start beaver log shipping at boot time
# Description:       Enable log shipping provided by beaver.
### END INIT INFO

PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
export PATH

BEAVER_NAME='beaver'
BEAVER_BIN="$(which "${BEAVER_NAME}")"
BEAVER_LOG='/var/log/beaver.log'
BEAVER_PID='/var/run/beaver.pid'
BEAVER_OPTS="-c /etc/beaver/beaver.conf -C /etc/beaver/conf.d -P ${BEAVER_PID} -l ${BEAVER_LOG} -D"
BEAVER_USER='beaver'

[ -r /etc/init.d/functions ] && . /etc/init.d/functions
[ -r /lib/lsb/init-functions ] && . /lib/lsb/init-functions

do_start() {
    test -f "${BEAVER_BIN}" || exit 0
    if is_up
    then
        echo $"${BEAVER_NAME} has already been started."
        return 0
    fi
    echo -n $"Starting ${BEAVER_NAME}"
    su - "${BEAVER_USER}" -s '/bin/bash' -c "${BEAVER_BIN} ${BEAVER_OPTS}"
    echo '.'
}

do_stop() {
    test -f "${BEAVER_BIN}" || exit 0
    if ! is_up
    then
        echo $"${BEAVER_NAME} has already been stopped."
        return 0
    fi
    echo -n $"Stopping ${BEAVER_NAME}"
    do_kill
    local I='0'
    while is_up
    do
        echo -n '.'
        if [ "${I}" -gt 10 ]
        then
            do_kill_force
            I='0'
        else
            do_kill
        fi
        sleep 1
        I="$((I+1))"
    done
    echo '.'
}

beaver_pid() {
    tail -1 "${BEAVER_PID}" 2> /dev/null
}

is_up() {
    PID="$(beaver_pid)"
    [ x"${PID}" != x ] && ps -p "${PID}" -o comm,args h 2> /dev/null | grep -qFw "${BEAVER_NAME}"
}

do_kill() {
    PID="$(beaver_pid)"
    [ x"${PID}" != x ] && su - "${BEAVER_USER}" -c "kill -TERM ${PID}"
}

do_kill_force() {
    PID="$(beaver_pid)"
    echo -n 'force'
    [ x"${PID}" != x ] && su - "${BEAVER_USER}" -c "kill -KILL ${PID}"
}

do_restart() {
    test -f "${BEAVER_BIN}" || exit 0
    do_stop
    sleep 1
    do_start
}

do_status() {
    test -f "${BEAVER_BIN}" || exit 0
    if is_up
    then
        echo "${BEAVER_NAME} is running."
        exit 0
    else
        echo "${BEAVER_NAME} is not running."
        exit 1
    fi
}

do_usage() {
    echo $"Usage: $0 {start | stop | restart | force-reload | status}"
    exit 1
}

case "$1" in
start)
    do_start
    exit "$?"
    ;;
stop)
    do_stop
    exit "$?"
    ;;
restart|force-reload)
    do_restart
    exit "$?"
    ;;
status)
    do_status
    ;;
*)
    do_usage
    ;;
esac