Skip to content

Instantly share code, notes, and snippets.

@hito-asa
Created March 29, 2012 06:45
Show Gist options
  • Save hito-asa/2234262 to your computer and use it in GitHub Desktop.
Save hito-asa/2234262 to your computer and use it in GitHub Desktop.
fluentd agent
<source>
type tail
path path_to_file1
pos_file path_to_pos_file1
tag app.file1
format /^(?<ip>[^ ]+) [^ ]+ [^ ]+ \[(?<time>[^\]]+)\] "(?<method>\S+) /(?<country>[A-Z0-9]{0,3})_(?<domain>[^/]+)/(?<module>[^/]+)/(?<controller>[^/]+)/(?<action>[^/]+)/(?<record>\S*)" (?<code>\d+) (?<res_time>\d+) "(?<referer>[^\"]*)" "(?<old_device>[^_\" ]*)_(?<device>[^_\" ]*)_(?<browser>[^_\" ]*) ?(?<guid>[^\"]*)" "(?<vuid>\d*) ?(?<option>[^\"]*)"$/
time_format %d/%b/%Y:%H:%M:%S %z
rotate_wait = 5s
</source>
<source>
type tail
path path_to_file2
pos_file path_to_pos_file1
tag app.file1
format /^[^ ]+ [^ ]+ [^ ]+ \[(?<time>[^\]]+)\] "\S+ /(?<module>[^/]+)/(?<controller>[^/]+)/(?<ad>[^\"]+)/" (?<code>\d+) (?<res_time>\d+) "(?<try_count>\d+)(?<result>[^\"]+)".*$/
time_format %d/%b/%Y:%H:%M:%S %z
rotate_wait = 5s
</source>
<source>
type tail
path path_to_file3
pos_file path_to_pos_file1
tag app.file1
format /(?<time>[^,]+,[^,]+),(?<res_time>\d+),(?<device>[^,]+),/(?<module>[^/]+)/(?<controller>[^/]+),(?<ad>[^\"]*)$/
time_format %Y/%m/%d,%H:%M:%S
rotate_wait = 5s
</source>
<match app.*>
type roundrobin
<store>
type forward
buffer_type memory
buffer_chunk_limit 256k
buffer_queue_limit 16
flush_interval 1s
retry_limit 5
retry_wait 1s
send_timeout 5s
recover_wait 5s
heartbeat_interval 1s
phi_threshold 10
hard_timeout 5s
<server>
name worker1
host 192.168.xxx.xxx
port 24224
weight 10
</server>
<server>
name worker2
host 192.168.xxx.xxx
port 24224
weight 10
</server>
...
</store>
<store>
...
</store>
...
</match>
#!/bin/bash
FLUENTD_HOME=/data/App/fluentd
PID_FILE=$FLUENTD_HOME/fluentd.pid
CONF_FILE=$FLUENTD_HOME/conf/fluentd.conf
LOG_FILE=$FLUENTD_HOME/logs/fluentd.out
PLUGIN_DIR=$FLUENTD_HOME/plugins
PSNAME="fluentd --daemon"
F_USER=xxxx
F_GROUP=xxxx
RUBY_VER="1.9.3-p125"
start()
{
PID=`pgrep -f "$PSNAME"`
if [ -z "$PID" ]; then
if [ -f $PID_FILE ]; then rm -f $PID_FILE; fi
else
echo "fluentd already started."
return -1
fi
if [ -z $rvm_path ]; then
export rvm_path=$FLUENTD_HOME/rvm
fi
source $rvm_path/scripts/rvm
rvm use $RUBY_VER 1> /dev/null
echo -n "Starting fluentd: "
fluentd --daemon $PID_FILE --user $F_USER --group $F_GROUP --config $CONF_FILE --plugin $PLUGIN_DIR --log $LOG_FILE -vv
echo "done."
}
stop()
{
PID=`pgrep -f "$PSNAME"`
if [ -z "$PID" ]; then
echo "fluentd already stopped."
return 0
fi
echo -n "Stopping fluentd: "
pkill -TERM -f "$PSNAME"
count=0
while [ ! -z "$PID" ]; do
count=`expr $count + 1`
if [ `expr $count % 10` == 0 ]; then pkill -KILL -f "$PSNAME"; fi
if [ $count == 60 ]; then
echo " failed."
return -1
fi
sleep 1
PID=`pgrep -f "$PSNAME"`
done
rm -f $PID_FILE
echo "done."
}
restart()
{
stop && start
}
update()
{
source $rvm_path/scripts/rvm
rvm use $RUBY_VER 1> /dev/null
gem update fluentd
}
case "$1" in
start)
start
;;
stop)
stop
;;
restart)
restart
;;
reload)
pkill -HUP -f "$PSNAME"
;;
condrestart)
if [ ! -f $PID_FILE ]; then
restart
elif [ ! -f /proc/`cat $PID_FILE`/status ]; then
restart
fi
;;
update)
stop
update
start
;;
*)
echo "Usage: $0 {start|stop|restart|reload}"
exit 1
;;
esac
exit $?
#!/bin/bash
if [ -d /data/App/fluentd/rvm/rubies ]; then
echo "/data/App/fluentd/rvm/rubies already exists."
exit -1
fi
mkdir /data/App/fluentd/rvm -p
export rvm_path=/data/App/fluentd/rvm
if [ -z "`grep '^insecure$' ~/.curlrc`" ]; then
echo insecure >> ~/.curlrc
fi
bash -s stable < <(curl -s https://raw.github.com/wayneeseguin/rvm/master/binscripts/rvm-installer)
sed -ie '/Add RVM/d' ~/.bashrc
sed -ie '/Load RVM/d' ~/.bash_profile
echo "export rvm_path=/data/App/fluentd/rvm" >> ~/.bash_profile
echo "[[ -s \"\$rvm_path/scripts/rvm\" ]] && source \"\$rvm_path/scripts/rvm\"" >> ~/.bash_profile
echo >> ~/.bash_profile
source ~/.bash_profile
rvm install 1.9.3-p125
rvm use 1.9.3-p125
gem update --system
gem install fluentd
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment