Skip to content

Instantly share code, notes, and snippets.

@sean-horn
Last active November 9, 2017 17:28
Show Gist options
  • Save sean-horn/52c42f70ecc88b18fd050489ff8ef12a to your computer and use it in GitHub Desktop.
Save sean-horn/52c42f70ecc88b18fd050489ff8ef12a to your computer and use it in GitHub Desktop.
automate-logstash-config
#!/bin/bash
usage="
This is a script used to configure your logstash tunables through delivery.rb, and to scale the number of \n
logstash processes running on your Automate server.\n
\n
You can specify the following options:\n
--workers The number of filter workers per logstash process. Defaults to number of virtual cores.\n
--total-procs The number of total logstash processes to run. Defaults to 2.\n
--heap-size The heap size allotted to each logstash process. Defaults to 2g.\n
--bulk-size The batch size for the bulk posts to ElasticSearch. Defaults to 25.\n
"
#if [ $# -eq 0 ]; then
# echo -e $usage
# exit 1
#fi
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
--workers)
LS_WORKERS="$2"
shift # past argument
;;
--heap-size)
LS_HEAP_SIZE="$2"
shift # past argument
;;
--total-procs)
LS_TOTAL_PROCS="$2"
shift # past argument
;;
--bulk-size)
LS_BULK_SIZE="$2"
shift
;;
-h|--help)
echo -e $usage
exit 0
;;
*)
echo "Unknown option $1"
echo -e $usage
exit 1
;;
esac
shift # past argument or value
done
if [ -z "$LS_HEAP_SIZE" ]; then
LS_HEAP_SIZE="2g"
fi
if [ -z "$LS_TOTAL_PROCS" ]; then
LS_TOTAL_PROCS="4"
fi
if [ -z "$LS_BULK_SIZE" ]; then
LS_BULK_SIZE="25"
fi
if [ -z "$LS_WORKERS" ]; then
LS_WORKERS="12"
fi
cat > /tmp/ls-config.rb <<EOF
ruby_block 'configure logstash settings in delivery.rb' do
block do
delivery_rb_lsconf = Chef::Util::FileEdit.new('/etc/delivery/delivery.rb')
unless delivery_rb_lsconf.search_file_replace_line(
/logstash\['config'\]/,
"logstash['config'] = { 'pipeline' => { 'batch' => { 'size' => $LS_BULK_SIZE }, 'workers' => $LS_WORKERS } }"
)
delivery_rb_lsconf.insert_line_if_no_match(
/logstash\['config'\]/,
"logstash['config'] = { 'pipeline' => { 'batch' => { 'size' => $LS_BULK_SIZE }, 'workers' => $LS_WORKERS } }"
)
end
delivery_rb_lsconf.write_file
delivery_rb_lsheap = Chef::Util::FileEdit.new('/etc/delivery/delivery.rb')
unless delivery_rb_lsheap.search_file_replace_line(
/logstash\['heap_size'\]/,
"logstash['heap_size'] = '$LS_HEAP_SIZE'"
)
delivery_rb_lsheap.insert_line_if_no_match(
/logstash\['heap_size'\]/,
"logstash['heap_size'] = '$LS_HEAP_SIZE'"
)
end
delivery_rb_lsheap.write_file
system('sudo automate-ctl reconfigure') if delivery_rb_lsconf.file_edited? || delivery_rb_lsheap.file_edited?
end
end
ruby_block 'reap extra logstash workers' do
block do
logstash_dirs = Dir['/opt/delivery/sv/logstash*']
if $LS_TOTAL_PROCS < logstash_dirs.length
($LS_TOTAL_PROCS + 1..logstash_dirs.length).each do |i|
system("sudo automate-ctl stop logstash#{i}")
FileUtils.rm_rf("/opt/delivery/sv/logstash#{i}")
FileUtils.rm_rf("/opt/delivery/embedded/etc/logstash/conf.d#{i}")
end
end
end
end
(2..$LS_TOTAL_PROCS).each do |i|
unless Dir.exist?("/opt/delivery/sv/logstash#{i}")
execute "copy logstash sv dir for logstash#{i}" do
command "cp -r /opt/delivery/sv/logstash/ /opt/delivery/sv/logstash#{i}"
end
execute "copy logstash conf dir for logstash#{i}" do
command "cp -r /opt/delivery/embedded/etc/logstash/conf.d /opt/delivery/embedded/etc/logstash/conf.d#{i}"
end
file "/opt/delivery/embedded/etc/logstash/conf.d#{i}/10-websocket-output.conf" do
action :delete
end
end
link "/opt/delivery/service/logstash#{i}" do
to "/opt/delivery/sv/logstash#{i}"
end
link "/opt/delivery/init/logstash#{i}" do
to '/opt/delivery/embedded/bin/sv'
end
directory "/var/log/delivery/logstash#{i}"
ruby_block "create directories for logstash#{i}" do
block do
ls_run_file = Chef::Util::FileEdit.new("/opt/delivery/sv/logstash#{i}/run")
ls_run_file.search_file_replace(/conf\.d /, "conf.d#{i} ")
ls_run_file.search_file_replace(/-w (?<!$LS_WORKERS)[0-9]+/, "-w $LS_WORKERS")
ls_run_file.search_file_replace(/-b (?<!$LS_BULK_SIZE)[0-9]+/, "-b $LS_BULK_SIZE")
ls_run_file.search_file_replace(/LS_HEAP_SIZE=(?<!$LS_HEAP_SIZE)[0-9]*m/, "LS_HEAP_SIZE=$LS_HEAP_SIZE")
ls_run_file.write_file
ls_log_run_file = Chef::Util::FileEdit.new("/opt/delivery/sv/logstash#{i}/log/run")
ls_log_run_file.search_file_replace(/logstash$/, "logstash#{i}")
ls_log_run_file.write_file
system("sudo automate-ctl restart logstash#{i}") if ls_run_file.file_edited? || ls_log_run_file.file_edited?
end
end
end
EOF
chef-apply /tmp/ls-config.rb
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment