Skip to content

Instantly share code, notes, and snippets.

@ergoz
Forked from trungv0/haproxy.sh
Created October 1, 2019 10:14
Show Gist options
  • Save ergoz/48a773c05667d688b7aa529b983d63c7 to your computer and use it in GitHub Desktop.
Save ergoz/48a773c05667d688b7aa529b983d63c7 to your computer and use it in GitHub Desktop.
RabbitMQ cluster with HAProxy & Keepalived for high availability
# install haproxy
yum install -y haproxy
# config haproxy for rabbitmq
cat > /etc/haproxy/haproxy.cfg << "EOF"
global
log 127.0.0.1 local0 notice
maxconn 10000
user haproxy
group haproxy
defaults
timeout connect 5s
timeout client 100s
timeout server 100s
listen rabbitmq
bind :5673
mode tcp
balance roundrobin
server rabbitmq-01 <node1>:5672 check inter 5s rise 2 fall 3
server rabbitmq-02 <node2>:5672 check inter 5s rise 2 fall 3
# optional, for proxying management site
frontend front_rabbitmq_management
bind :15672
default_backend back_rabbitmq_management
backend back_rabbitmq_management
balance source
server rabbitmq-mgmt-01 10.25.1.101:15673 check
server rabbitmq-mgmt-02 10.25.1.102:15673 check
# optional, for monitoring
listen stats :9000
mode http
stats enable
stats hide-version
stats realm Haproxy\ Statistics
stats uri /
stats auth haproxy:haproxy
EOF
# restart haproxy
systemctl restart haproxy
# TODO haproxy logging
# install keepalived
yum install -y keepalived
# config keepalived for rotating & high availability virtual ip
cat > /etc/keepalived/keepalived.cfg << "EOF"
vrrp_script chk_haproxy {
script "killall -0 haproxy" # health check script, will force master election when error code is yielded
interval 2
weight 5
}
vrrp_instance VI_1 {
interface eth0
state MASTER # or BACKUP for backup instances
virtual_router_id 202 # must be synchronized across instances
priority 101 # must vary across instances, highest priority instance is master
# must ensure that (priority + health check weight) of lowest priority instance
# be high enough to outweight highest priority amongst instances
advert_int 1
unicast_src_ip <node1> # ip of the instance
unicast_peer {
<node2> # ip of other instances, add more ip as necessary
}
// must be the same for all instances
authentication {
auth_type PASS
auth_pass password
}
# virtual ip to bind to master instance
virtual_ipaddress {
<virtual ip>
}
# call check script defined above
track_script {
chk_haproxy
}
}
EOF
# restart keepalived
systemctl restart keepalived
## basic installations
# add erlang repo
cat > /etc/yum.repos.d/esl-erlang.repo << "EOF"
[erlang-solutions]
name=Centos $releasever - $basearch - Erlang Solutions
baseurl=http://binaries.erlang-solutions.com/rpm/centos/$releasever/$basearch
gpgcheck=1
gpgkey=http://binaries.erlang-solutions.com/debian/erlang_solutions.asc
enabled=1
EOF
# install erlang
yum install erlang
# import rabbitmq key
rpm --import https://www.rabbitmq.com/rabbitmq-signing-key-public.asc
# download installer
wget https://www.rabbitmq.com/releases/rabbitmq-server/v3.6.1/rabbitmq-server-3.6.1-1.noarch.rpm
# install rabbitmq
yum install rabbitmq-server-3.6.1-1.noarch.rpm
# ensure rabbitmq-server start on system boot
chkconfig rabbitmq-server on
service rabbitmq-server start
# check
rabbitmqctl status
# enable management plugin
rabbitmq-plugins enable rabbitmq_management
# add user (admin)
rabbitmqctl add_user admin password
rabbitmqctl set_permissions admin '.*' '.*' '.*'
rabbitmqctl set_user_tags admin administrator
# restart rabbitmq
service rabbitmq-server restart
## how to: cluster
# add hosts to all cluster nodes, so they know how to reach each other
# retrieve erlang cookie of a node
cat /var/lib/rabbitmq/.erlang.cookie
# synchronize that value to any other nodes of the cluster
cat > /var/lib/rabbitmq/.erlang.cookie << 'the cookie'
rabbitmqctl stop_app
# join all nodes to one to form a cluster
rabbitmqctl join_cluster rabbit@<node-hostname>
rabbitmqctl cluster_status
## how to: tune
cat > /etc/rabbitmq/rabbitmq.config << "EOF"
[
{rabbit, [
{tcp_listeners, [{"0.0.0.0", 5672}]},
{vm_memory_high_watermark, 0.9},{vm_memory_high_watermark_paging_ratio, 0.85}
]}
].
EOF
#
vi /etc/sysctl.conf
```
# General gigabit tuning:
net.core.rmem_max = 8738000
net.core.wmem_max = 6553600
net.ipv4.tcp_rmem = 8192 873800 8738000
net.ipv4.tcp_wmem = 4096 655360 6553600
# VERY important to reuse ports in TCP_WAIT
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_tw_buckets = 360000
net.core.netdev_max_backlog = 2500
vm.min_free_kbytes = 65536
vm.swappiness = 0
fs.file-max = 655360
```
# apply change
sysctl -p
/etc/init.d/rabbitmq-server restart
# set policies (ttl) for all queues
rabbitmqctl set_policy TTL ".*" '{"message-ttl":1800000}' --apply-to queues
## how to: monitor
wget http://127.0.0.1:15672/cli/rabbitmqadmin
mv rabbitmqadmin /usr/local/bin/
chmod 755 /usr/local/bin/rabbitmqadmin
# try it
rabbitmqadmin list exchanges
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment