Last active
November 9, 2017 17:28
-
-
Save sean-horn/52c42f70ecc88b18fd050489ff8ef12a to your computer and use it in GitHub Desktop.
automate-logstash-config
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
usage=" | |
This is a script used to configure your logstash tunables through delivery.rb, and to scale the number of \n | |
logstash processes running on your Automate server.\n | |
\n | |
You can specify the following options:\n | |
--workers The number of filter workers per logstash process. Defaults to number of virtual cores.\n | |
--total-procs The number of total logstash processes to run. Defaults to 2.\n | |
--heap-size The heap size allotted to each logstash process. Defaults to 2g.\n | |
--bulk-size The batch size for the bulk posts to ElasticSearch. Defaults to 25.\n | |
" | |
#if [ $# -eq 0 ]; then | |
# echo -e $usage | |
# exit 1 | |
#fi | |
while [[ $# -gt 0 ]] | |
do | |
key="$1" | |
case $key in | |
--workers) | |
LS_WORKERS="$2" | |
shift # past argument | |
;; | |
--heap-size) | |
LS_HEAP_SIZE="$2" | |
shift # past argument | |
;; | |
--total-procs) | |
LS_TOTAL_PROCS="$2" | |
shift # past argument | |
;; | |
--bulk-size) | |
LS_BULK_SIZE="$2" | |
shift | |
;; | |
-h|--help) | |
echo -e $usage | |
exit 0 | |
;; | |
*) | |
echo "Unknown option $1" | |
echo -e $usage | |
exit 1 | |
;; | |
esac | |
shift # past argument or value | |
done | |
if [ -z "$LS_HEAP_SIZE" ]; then | |
LS_HEAP_SIZE="2g" | |
fi | |
if [ -z "$LS_TOTAL_PROCS" ]; then | |
LS_TOTAL_PROCS="4" | |
fi | |
if [ -z "$LS_BULK_SIZE" ]; then | |
LS_BULK_SIZE="25" | |
fi | |
if [ -z "$LS_WORKERS" ]; then | |
LS_WORKERS="12" | |
fi | |
cat > /tmp/ls-config.rb <<EOF | |
ruby_block 'configure logstash settings in delivery.rb' do | |
block do | |
delivery_rb_lsconf = Chef::Util::FileEdit.new('/etc/delivery/delivery.rb') | |
unless delivery_rb_lsconf.search_file_replace_line( | |
/logstash\['config'\]/, | |
"logstash['config'] = { 'pipeline' => { 'batch' => { 'size' => $LS_BULK_SIZE }, 'workers' => $LS_WORKERS } }" | |
) | |
delivery_rb_lsconf.insert_line_if_no_match( | |
/logstash\['config'\]/, | |
"logstash['config'] = { 'pipeline' => { 'batch' => { 'size' => $LS_BULK_SIZE }, 'workers' => $LS_WORKERS } }" | |
) | |
end | |
delivery_rb_lsconf.write_file | |
delivery_rb_lsheap = Chef::Util::FileEdit.new('/etc/delivery/delivery.rb') | |
unless delivery_rb_lsheap.search_file_replace_line( | |
/logstash\['heap_size'\]/, | |
"logstash['heap_size'] = '$LS_HEAP_SIZE'" | |
) | |
delivery_rb_lsheap.insert_line_if_no_match( | |
/logstash\['heap_size'\]/, | |
"logstash['heap_size'] = '$LS_HEAP_SIZE'" | |
) | |
end | |
delivery_rb_lsheap.write_file | |
system('sudo automate-ctl reconfigure') if delivery_rb_lsconf.file_edited? || delivery_rb_lsheap.file_edited? | |
end | |
end | |
ruby_block 'reap extra logstash workers' do | |
block do | |
logstash_dirs = Dir['/opt/delivery/sv/logstash*'] | |
if $LS_TOTAL_PROCS < logstash_dirs.length | |
($LS_TOTAL_PROCS + 1..logstash_dirs.length).each do |i| | |
system("sudo automate-ctl stop logstash#{i}") | |
FileUtils.rm_rf("/opt/delivery/sv/logstash#{i}") | |
FileUtils.rm_rf("/opt/delivery/embedded/etc/logstash/conf.d#{i}") | |
end | |
end | |
end | |
end | |
(2..$LS_TOTAL_PROCS).each do |i| | |
unless Dir.exist?("/opt/delivery/sv/logstash#{i}") | |
execute "copy logstash sv dir for logstash#{i}" do | |
command "cp -r /opt/delivery/sv/logstash/ /opt/delivery/sv/logstash#{i}" | |
end | |
execute "copy logstash conf dir for logstash#{i}" do | |
command "cp -r /opt/delivery/embedded/etc/logstash/conf.d /opt/delivery/embedded/etc/logstash/conf.d#{i}" | |
end | |
file "/opt/delivery/embedded/etc/logstash/conf.d#{i}/10-websocket-output.conf" do | |
action :delete | |
end | |
end | |
link "/opt/delivery/service/logstash#{i}" do | |
to "/opt/delivery/sv/logstash#{i}" | |
end | |
link "/opt/delivery/init/logstash#{i}" do | |
to '/opt/delivery/embedded/bin/sv' | |
end | |
directory "/var/log/delivery/logstash#{i}" | |
ruby_block "create directories for logstash#{i}" do | |
block do | |
ls_run_file = Chef::Util::FileEdit.new("/opt/delivery/sv/logstash#{i}/run") | |
ls_run_file.search_file_replace(/conf\.d /, "conf.d#{i} ") | |
ls_run_file.search_file_replace(/-w (?<!$LS_WORKERS)[0-9]+/, "-w $LS_WORKERS") | |
ls_run_file.search_file_replace(/-b (?<!$LS_BULK_SIZE)[0-9]+/, "-b $LS_BULK_SIZE") | |
ls_run_file.search_file_replace(/LS_HEAP_SIZE=(?<!$LS_HEAP_SIZE)[0-9]*m/, "LS_HEAP_SIZE=$LS_HEAP_SIZE") | |
ls_run_file.write_file | |
ls_log_run_file = Chef::Util::FileEdit.new("/opt/delivery/sv/logstash#{i}/log/run") | |
ls_log_run_file.search_file_replace(/logstash$/, "logstash#{i}") | |
ls_log_run_file.write_file | |
system("sudo automate-ctl restart logstash#{i}") if ls_run_file.file_edited? || ls_log_run_file.file_edited? | |
end | |
end | |
end | |
EOF | |
chef-apply /tmp/ls-config.rb |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment