Custom Flume NG Agent INIT script for centos for runnig multiple agents on same machine

flume-ng-agent.sh

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Starts a Flume agent
#
# chkconfig: 345 90 10
# description: Flume agent
#
### BEGIN INIT INFO
# Provides:          flume-ng-agent
# Required-Start:    $remote_fs
# Should-Start:
# Required-Stop:     $remote_fs
# Should-Stop:
# Default-Start:     3 4 5
# Default-Stop:      0 1 2 6
# Short-Description: Flume agent
### END INIT INFO

. /lib/lsb/init-functions

# Name of the agnet
FLUME_AGENT_NAME=a1

# Setting up a few defaults that can be later overrideen in /etc/default/flume-ng-agent
FLUME_LOG_DIR=/var/log/flume-ng
FLUME_CONF_DIR=/etc/flume-ng/conf_${FLUME_AGENT_NAME}
FLUME_RUN_DIR=/var/run/flume-ng
FLUME_HOME=/usr/lib/flume-ng
FLUME_USER=flume

# Autodetect JAVA_HOME if not defined
if [ -e /usr/libexec/bigtop-detect-javahome ]; then
  . /usr/libexec/bigtop-detect-javahome
elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
  . /usr/lib/bigtop-utils/bigtop-detect-javahome
fi

STATUS_RUNNING=0
STATUS_DEAD=1
STATUS_DEAD_AND_LOCK=2
STATUS_NOT_RUNNING=3

ERROR_PROGRAM_NOT_INSTALLED=5

FLUME_LOCK_DIR="/var/lock/subsys/"
LOCKFILE="${FLUME_LOCK_DIR}/flume-ng-agent"
desc="Flume agent daemon"

FLUME_CONF_FILE=${FLUME_CONF_FILE:-${FLUME_CONF_DIR}/flume.conf}
EXEC_PATH=/usr/bin/flume-ng
FLUME_PID_FILE=${FLUME_RUN_DIR}/flume-ng-agent-${FLUME_AGENT_NAME}.pid

# These directories may be tmpfs and may or may not exist
# depending on the OS (ex: /var/lock/subsys does not exist on debian/ubuntu)
for dir in "$FLUME_RUN_DIR" "$FLUME_LOCK_DIR"; do
  [ -d "${dir}" ] || install -d -m 0755 -o $FLUME_USER -g $FLUME_USER ${dir}
done

FLUME_SHUTDOWN_TIMEOUT=${FLUME_SHUTDOWN_TIMEOUT:-60}

start() {
  [ -x $exec ] || exit $ERROR_PROGRAM_NOT_INSTALLED

  checkstatus
  status=$?
  if [ "$status" -eq "$STATUS_RUNNING" ]; then
    exit 0
  fi

  log_success_msg "Starting $desc (flume-ng-agent): "
  /bin/su -s /bin/bash -c "/bin/bash -c 'echo \$\$ > ${FLUME_PID_FILE} && exec ${EXEC_PATH} agent --conf $FLUME_CONF_DIR --conf-file $FLUME_CONF_FILE --name $FLUME_AGENT_NAME >>${FLUME_LOG_DIR}/flume.${FLUME_AGENT_NAME}.init.log 2>&1 ' &" $FLUME_USER
  RETVAL=$?
  [ $RETVAL -eq 0 ] && touch $LOCKFILE
  return $RETVAL
}

stop() {
  if [ ! -e $FLUME_PID_FILE ]; then
    log_failure_msg "Flume agent is not running"
    exit 0
  fi

  log_success_msg "Stopping $desc (flume-ng-agent): "

  FLUME_PID=`cat $FLUME_PID_FILE`
  if [ -n $FLUME_PID ]; then
    kill -TERM ${FLUME_PID} &>/dev/null
    for i in `seq 1 ${FLUME_SHUTDOWN_TIMEOUT}` ; do
      kill -0 ${FLUME_PID} &>/dev/null || break
      sleep 1
    done
    kill -KILL ${FLUME_PID} &>/dev/null
  fi
  rm -f $LOCKFILE $FLUME_PID_FILE
  return 0
}

restart() {
  stop
  start
}

checkstatus(){
  pidofproc -p $FLUME_PID_FILE java > /dev/null
  status=$?

  case "$status" in
    $STATUS_RUNNING)
      log_success_msg "Flume agent is running"
      ;;
    $STATUS_DEAD)
      log_failure_msg "Flume agent is dead and pid file exists"
      ;;
    $STATUS_DEAD_AND_LOCK)
      log_failure_msg "Flume agent is dead and lock file exists"
      ;;
    $STATUS_NOT_RUNNING)
      log_failure_msg "Flume agent is not running"
      ;;
    *)
      log_failure_msg "Flume agent status is unknown"
      ;;
  esac
  return $status
}

condrestart(){
  [ -e ${LOCKFILE} ] && restart || :
}

case "$1" in
  start)
    start
    ;;
  stop)
    stop
    ;;
  status)
    checkstatus
    ;;
  restart)
    restart
    ;;
  condrestart|try-restart)
    condrestart
    ;;
  *)
    echo $"Usage: $0 {start|stop|status|restart|try-restart|condrestart}"
    exit 1
esac

exit $RETVAL

usage.md

To use this script, you should rename the FLUME_AGENT_NAME in the script to then name of the flume agent also, create a seperate flume conf dir like this:

cp -LR /etc/flume-ng/conf /etc/flume-ng/conf_FLUME_AGENT_NAME

where FLUME_AGENT_NAME is the name of the flume agent and edit the following files inside /etc/flume-ng/conf_FLUME_AGENT_NAME

1. /etc/flume-ng/conf_FLUME_AGENT_NAME/flume.conf
2. /etc/flume-ng/conf_FLUME_AGENT_NAME/log4j.properties - flume.log.file to FLUME_AGENT_NAME

Great script, thanks for sharing. I made a fork with some slight adjustments and some more elaborate explanation on installing Flume.

sorry, it's not clear to me how this supports multiple agents on the same machine, as the description indicates. The only way I can tell to do that from this script would be to copy multiple instances of this .sh script (unique names, obviously) and launch them separately.

Maybe I'm missing something? Otherwise it looks like the standard single agent script.

If the PID is blocked on IO or -STOPed line#107 will not actually kill the process, yet nothing checks again afterwards to see if its gone, and removes the lock/pid file, leaving the script free to start a second flume which will fail to bind the port but still run.