Setup LogStash on test server

@LogStash

.

_init_elasticsearch.conf

# ref: https://gist.github.com/rbscott/1052015
 
description "ElasticSearch"
 
start on (net-device-up
and local-filesystems
and runlevel [2345])
 
stop on runlevel [016]
 
respawn limit 10 5
limit nofile 32000 32000
 
# store data under ES_HOME/data
env ES_HOME=/mnt/proC/elasticsearch/
env ES_MIN_MEM=256m
env ES_MAX_MEM=700m
 
console log
 
exec $ES_HOME/bin/elasticsearch -f -Des.max-open-files=true

_init_logstash.conf

description "logstash archiver"

start on virtual-filesystems
stop on runlevel [06]

respawn
limit nofile 65550 65550
console log

setuid liukun
env SINCEDB_DIR="/home/liukun/tmp"
exec /home/liukun/app/logstash-1.4.0/bin/logstash agent -f /home/liukun/proCconf/logstash-local.conf

_install.sh

curl -O https://download.elasticsearch.org/logstash/logstash/logstash-1.4.0.tar.gz
tar zxvf logstash-1.4.0.tar.gz
cd logstash-1.4.0
# test
bin/logstash -e 'input { stdin { } } output { stdout {} }'

# better match ElasticSearch version with LogStash
curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.0.1.tar.gz
tar zxvf elasticsearch-1.0.1.tar.gz
cd elasticsearch-1.0.1/
# test
./bin/elasticsearch

# by default, elasticsearch store data under sub-directory

kibana.nginx

# https://github.com/elasticsearch/kibana/blob/master/sample/nginx.conf

##
# You should look at the following URL's in order to grasp a solid understanding
# of Nginx configuration files in order to fully unleash the power of Nginx.
# http://wiki.nginx.org/Pitfalls
# http://wiki.nginx.org/QuickStart
# http://wiki.nginx.org/Configuration
#
# Generally, you will want to move this file somewhere, and start with a clean
# file but keep this around for reference. Or just disable in sites-enabled.
#
# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples.
##

server {
	root /usr/share/nginx/www;
	index index.html index.htm;
	#auth_basic "Restricted";
	#auth_basic_user_file /usr/share/nginx/www/.htpasswd;

	# Make site accessible from http://localhost/
	server_name localhost;

	location / {
		# First attempt to serve request as file, then
		# as directory, then fall back to index.html
		try_files $uri $uri/ /index.html;
		# Uncomment to enable naxsi on this location
		# include /etc/nginx/naxsi.rules
	}

	location /log {
		alias /home/liukun/app/logstash-1.4.0/vendor/kibana;
		index  index.html;
	}

  location ~ ^/_aliases$ {
    proxy_pass http://127.0.0.1:9200;
    proxy_read_timeout 90;
  }
  location ~ ^/.*/_aliases$ {
    proxy_pass http://127.0.0.1:9200;
    proxy_read_timeout 90;
  }
  location ~ ^/_nodes$ {
    proxy_pass http://127.0.0.1:9200;
    proxy_read_timeout 90;
  }
  location ~ ^/.*/_search$ {
    proxy_pass http://127.0.0.1:9200;
    proxy_read_timeout 90;
  }
  location ~ ^/.*/_mapping {
    proxy_pass http://127.0.0.1:9200;
    proxy_read_timeout 90;
  }

  location ~ ^/kibana-int/dashboard/.*$ {
    proxy_pass http://127.0.0.1:9200;
    proxy_read_timeout 90;
  }
  location ~ ^/kibana-int/temp.*$ {
    proxy_pass http://127.0.0.1:9200;
    proxy_read_timeout 90;
  }

	# Only for nginx-naxsi : process denied requests
	#location /RequestDenied {
		# For example, return an error code
		#return 418;
	#}

	#error_page 404 /404.html;

	# redirect server error pages to the static page /50x.html
	#
	#error_page 500 502 503 504 /50x.html;
	#location = /50x.html {
	#	root /usr/share/nginx/www;
	#}

	# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
	#
	#location ~ \.php$ {
	#	fastcgi_split_path_info ^(.+\.php)(/.+)$;
	#	# NOTE: You should have "cgi.fix_pathinfo = 0;" in php.ini
	#
	#	# With php5-cgi alone:
	#	fastcgi_pass 127.0.0.1:9000;
	#	# With php5-fpm:
	#	fastcgi_pass unix:/var/run/php5-fpm.sock;
	#	fastcgi_index index.php;
	#	include fastcgi_params;
	#}

	# deny access to .htaccess files, if Apache's document root
	# concurs with nginx's one
	#
	#location ~ /\.ht {
	#	deny all;
	#}
}


# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
#	listen 8000;
#	listen somename:8080;
#	server_name somename alias another.alias;
#	root html;
#	index index.html index.htm;
#
#	location / {
#		try_files $uri $uri/ /index.html;
#	}
#}


# HTTPS server
#
#server {
#	listen 443;
#	server_name localhost;
#
#	root html;
#	index index.html index.htm;
#
#	ssl on;
#	ssl_certificate cert.pem;
#	ssl_certificate_key cert.key;
#
#	ssl_session_timeout 5m;
#
#	ssl_protocols SSLv3 TLSv1;
#	ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv3:+EXP;
#	ssl_prefer_server_ciphers on;
#
#	location / {
#		try_files $uri $uri/ /index.html;
#	}
#}

logstash.conf

# http://logstash.net/docs/1.4.2/
input {
#  file {
#    codec => json
#    path => [ "/home/liukun/log/game*.log", "/home/liukun/log/user*.log" ]
#    sincedb_path => "/home/liukun/log/logstash.sincedb"
#  }
  udp {
    codec => json
    port => 18000
  }
}
#filter {
#  if [level] < 30 { # drop log below INFO
#    drop {}
#  }
#  date {
#    match => [ "time", "ISO8601" ]
#    remove_field => "time"
#  }
#}
output {
  file {
    codec => json
    path => "/mnt/proC/log/four_%{+YYYY-MM-dd}.log"
  }
#  elasticsearch { host => localhost }
}