Notes for setting up logstash-forwarder as a log shipper to a RPC logstash server

install-logstash-forwarder.sh

###
### On the Logstash Server
###

# Prepare the server certificate directories
CERT_DIR='/etc/pki/tls'
mkdir -p ${CERT_DIR}/certs
mkdir ${CERT_DIR}/private

# Allow the server IP to be in the certificate subjectAltName
IP_ADDRESS=$(ip addr show dev eth1 | awk '/inet / {print $2}' | cut -d/ -f1)
sed -i "/\[ v3_ca \]/ a\
subjectAltName = IP: ${IP_ADDRESS}" /etc/ssl/openssl.cnf

# Generate the IP address based certificate
openssl req -config /etc/ssl/openssl.cnf -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout ${CERT_DIR}/private/logstash-forwarder.key -out ${CERT_DIR}/certs/logstash-forwarder.crt

# Setup the lumberjack listener
cat >/etc/logstash/conf.d/01-input.conf <<EOL
#===============================================================================
input {
    syslog {
        port => 5544
        codec => json
    }
    lumberjack {
        port => 5000
        type => "logs"
        ssl_certificate => "${CERT_DIR}/certs/logstash-forwarder.crt"
        ssl_key => "${CERT_DIR}/private/logstash-forwarder.key"
    }
}
#===============================================================================
EOL

# Restart Logstash
service logstash restart

# Copy the server's public certificate to the Logstash Client
CLIENT_IP='172.29.238.4'
scp /etc/pki/tls/certs/logstash-forwarder.crt root@${CLIENT_IP}:/tmp

###
### On the Logstash Client
###

# Copy the Logstash Server certificate into place
mkdir -p /etc/pki/tls/certs
mv /tmp/logstash-forwarder.crt /etc/pki/tls/certs/

# Add the apt repo key
wget -O - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add -

# Add the apt repo
echo 'deb http://packages.elasticsearch.org/logstashforwarder/debian stable main' > /etc/apt/sources.list.d/logstash-forwarder.list

# Update the apt cache and install the logstash forwarder
apt-get update && apt-get install logstash-forwarder

# Determine the Logstash Client IP Address
IP_ADDRESS=$(ip addr show dev eth1 | awk '/inet / {print $2}' | cut -d/ -f1)

# Set the Logstash Server IP Address
LOGSTASH_IP='172.29.236.207'

# Configure the Logstash forwarder
cat >/etc/logstash-forwarder.conf <<EOL
{
  "network": {
    "servers": [ "${LOGSTASH_IP}:5000" ],
    "ssl ca": "/etc/pki/tls/certs/logstash-forwarder.crt",
    "timeout": 15
  },
  "files": [
    {
      "paths": [
        "/var/log/log-storage/*/*.log"
      ],
      "fields": { "host": "${IP_ADDRESS}" }
    }
  ]
}
EOL

# Restart the logstash forwarder to start processing logs
service logstash-forwarder restart