README.md

Installing Logstash 1.4.0, ElasticSearch 1.1.0, and Kibana 3 on Ubuntu 12.04

• indexer and shipper setups
• TODO: add auth layer between kibana and elasticsearch... https://github.com/fangli/kibana-authentication-proxy ?

References

• http://logstash.net/docs/1.4.0/tutorials/getting-started-with-logstash
• https://github.com/elasticsearch/kibana
• http://www.webupd8.org/2012/09/install-oracle-java-8-in-ubuntu-via-ppa.html
• https://gist.github.com/wingdspur/2026107 <-- elastic search install script, nice!
• http://cookbook.logstash.net/rsyslog
• http://cookbook.logstash.net/recipes/statsd-metrics/
• http://rashidkpc.github.io/Kibana/infrastructure.html
• http://cookbook.logstash.net/recipes/logging-from-nodejs/
• https://www.npmjs.org/package/bucker
• http://collectd.org/features.shtml
• http://michael.bouvy.net/blog/en/2013/12/06/use-lumberjack-logstash-forwarder-to-forward-logs-logstash/
• http://logstash.net/docs/1.4.0/inputs/lumberjack

indexer_setup.sh

# be root for this

# enable auto accept oracle license
echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections

# install java 8 via ppa
add-apt-repository ppa:webupd8team/java
apt-get update
apt-get -y install oracle-java8-installer oracle-java8-set-default supervisor rsyslog vim unzip nginx

# install logstash to /opt/logstash
curl -O https://download.elasticsearch.org/logstash/logstash/logstash-1.4.0.tar.gz
tar zxvf logstash-1.4.0.tar.gz
mv logstash-1.4.0 /opt/logstash
rm logstash-1.4.0.tar.gz

# install elasticsearch
wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.1.0.deb
dpkg -i elasticsearch-1.1.0.deb
update-rc.d elasticsearch defaults 95 10
service elasticsearch start
ufw allow 9200

# install nodejs for kibana
curl -o ~/node.tar.gz http://nodejs.org/dist/v0.10.24/node-v0.10.24-linux-x64.tar.gz
cd /usr/local && tar --strip-components 1 -xzf ~/node.tar.gz && rm ~/node.tar.gz

# install kibana
wget http://download.elasticsearch.org/kibana/kibana/kibana-latest.zip
unzip kibana-latest.zip -d /opt
mv /opt/kibana-latest /opt/kibana
cd /opt/kibana

# setup elasticsearch URL
vim config.js

# serve kibana with nginx
rm -rf /usr/share/nginx/html
ln -s /opt/kibana /usr/share/nginx/html
ufw allow 80

# setup lumberjack support
cd /etc/init.d/
wget https://raw.github.com/elasticsearch/logstash-forwarder/master/logstash-forwarder.init -O logstash-forwarder

# setup lumberjack security
openssl req -x509 -batch -nodes -newkey rsa:2048 -keyout logstash-forwarder.key -out logstash-forwarder.crt
mkdir -p /etc/ssl/private
mkdir -p /etc/ssl/certs
mv logstash-forwarder.key /etc/ssl/private
mv logstash-forwarder.crt /etc/ssl/certs

# configure logstash
mkdir /etc/logstash
cat << EOF > /etc/logstash/logstash.conf
input {
  lumberjack {
    # The port to listen on
    port => 5043

    # The paths to your ssl cert and key
    ssl_certificate => "/etc/ssl/certs/logstash-forwarder.crt"
    ssl_key => "/etc/ssl/private/logstash-forwarder.key"

    # Set this to whatever you want.
    type => "lumberjack"
  }
}

output {
  elasticsearch { host => localhost }
}
EOF

# allow logstash forwarder packets
ufw allow 5043

# configure logstash to run on startup with supervisor
cat << EOF > /etc/supervisor/conf.d/logstash.conf
[program:logstash]
directory = /opt/logstash
command = /opt/logstash/bin/logstash --config /etc/logstash/logstash.conf
stdout_logfile = /var/log/supervisor/%(program_name)s.log
stderr_logfile = /var/log/supervisor/%(program_name)s.log
autostart = true
autorestart = true
EOF

tar -czf ~/ssl.tar.gz /etc/ssl

cat << EOF | echo
Indexer is ready.
  1) Transfer the SSL certs with the following command: 
    scp ~/ssl.tar.gz root@host:/root
  2) Configure the shipper using shipper_setup.sh
EOF

shipper_setup.sh

# be root

tar -zvxf ssl.tar.gz
mv etc/ssl/private/logstash-forwarder.key /etc/ssl/private
mv etc/ssl/certs/logstash-forwarder.crt /etc/ssl/certs
rm -rf etc ssl.tar.gz

# configure the script
logstash_ip="192.168.122.190"

# get stuff
wget -O - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add -
echo "deb http://packages.elasticsearch.org/logstashforwarder/debian stable main" >> /etc/apt/sources.list
apt-get update
apt-get -y install logstash-forwarder
update-rc.d logstash-forwarder defaults

# configure logstash-forwarder
cat << EOF > /etc/logstash-forwarder
{
  "network": {
    "servers": [ "$logstash_ip:5043" ],
    "ssl certificate": "/etc/ssl/certs/logstash-forwarder.crt",
    "ssl key": "/etc/ssl/private/logstash-forwarder.key",
    "ssl ca": "/etc/ssl/certs/logstash-forwarder.crt"
  },
  "files": [
    {
      "paths": [
        "/var/log/syslog",
      ]
      "fields": { "type": "syslog" }
    },
    {
      "paths": [
        "/var/log/nginx/*access.log",
        "/var/log/nginx/*error.log"
      ],
      "fields": { "type": "nginx" }
    }
  ]
}
EOF

sudo service logstash-forwarder start