Galaxy w/ Supervisor uWSGI Zerg Mode

00_supervisor_conf.ini

; /etc/supervisor/conf.d/galaxy.conf

; comment [program:galaxy] and [group:gx] and add:

[program:zergpool]
command         = uwsgi --plugin zergpool --master --zerg-pool /srv/galaxy/var/zergpool.sock:127.0.0.1:4001 --logto /srv/galaxy/log/zergpool.log
directory       = /srv/galaxy/server
priority        = 899
umask           = 022
autostart       = true
autorestart     = true
startsecs       = 5
user            = galaxy
environment     = HOME="/srv/galaxy",VIRTUAL_ENV="/srv/galaxy/venv",PATH="/srv/galaxy/venv/bin:%(ENV_PATH)s"
numprocs        = 1
stopsignal      = INT

[program:zergling]
command         = uwsgi --plugin python --virtualenv /srv/galaxy/venv --ini-paste /srv/galaxy/config/galaxy.ini --stats 127.0.0.1:919%(process_num)s --logto /srv/galaxy/log/zergling%(process_num)s.log
directory       = /srv/galaxy/server
process_name    = zergling%(process_num)s
priority        = 999
umask           = 022
autostart       = true
autorestart     = true
startsecs       = 15
user            = galaxy
environment     = HOME="/srv/galaxy",VIRTUAL_ENV="/srv/galaxy/venv",PATH="/srv/galaxy/venv/bin:%(ENV_PATH)s",DRMAA_LIBRARY_PATH="/usr/lib/slurm-drmaa/lib/libdrmaa.so.1"
numprocs        = 2
stopsignal      = INT

[group:gx]
programs = zergpool,zergling,handler

01_galaxy.ini

; comment `logto` in [uwsgi] section and add the following to the bottom of [uwsgi]

; zerg mode
; http://lists.unbit.it/pipermail/uwsgi/2014-October/007683.html
; https://gist.githubusercontent.com/unbit/2674313f070673a720e3/raw/56c804136c917ce1204b656f2d46e9988b48b1c7/spinningfifo.c
; this plugin was manually built into the test/main virtualenvs with:
;   uwsgi --build-plugin spinningfifo.c
;   mkdir -p /srv/galaxy/venv/lib/uwsgi/plugins
;   mv spinningfifo_plugin.so /srv/galaxy/venv/lib/uwsgi/plugins
plugins-dir = /srv/galaxy/venv/lib/uwsgi/plugins
plugin = spinningfifo

; fifo '0'
master-fifo = /srv/galaxy/var/zerg-new.fifo
; fifo '1'
master-fifo = /srv/galaxy/var/zerg-run.fifo
; fifo '2'
master-fifo = /srv/galaxy/var/zerg-old.fifo

; attach to zerg
zerg = /srv/galaxy/var/zergpool.sock

; we do not kill the old sleeping instance since supervisor is managing the start/stop
; force the currently running instance to became sleeping (slot 2) and place it in pause mode
if-exists = /srv/galaxy/var/zerg-run.fifo
  hook-accepting1-once = writefifo:/srv/galaxy/var/zerg-run.fifo 2p
endif =
; force this instance to became the running one (slot 1)
hook-accepting1-once = spinningfifo:/srv/galaxy/var/zerg-new.fifo 1

; perform restarts with `echo q >/srv/galaxy/var/zerg-old.fifo`

02_commands.console

sudo -u galaxy mkdir /srv/galaxy/var
sudo apt install -y uuid-dev libcap-dev libpcre3-dev
curl -LO https://gist.githubusercontent.com/unbit/2674313f070673a720e3/raw/56c804136c917ce1204b656f2d46e9988b48b1c7/spinningfifo.c
uwsgi --build-plugin spinningfifo.c
sudo -u galaxy mkdir -p /srv/galaxy/venv/lib/uwsgi/plugins
sudo mv spinningfifo_plugin.so /srv/galaxy/venv/lib/uwsgi/plugins
sudo chown galaxy:galaxy /srv/galaxy/venv/lib/uwsgi/plugins/spinningfifo_plugin.so
sudo -Hu galaxy /srv/galaxy/venv/bin/pip install uwsgitop
while [ 1 ]; do sudo -Hu galaxy /srv/galaxy/venv/bin/uwsgitop localhost:9190; sleep 1; done
while [ 1 ]; do sudo -Hu galaxy /srv/galaxy/venv/bin/uwsgitop localhost:9191; sleep 1; done
sudo supervisorctl update
# caveat: race condition for opening zergling sockets causes both zerglings to start as "active", fix by restarting one zergling:
sudo supervisorctl restart gx:zergling0
# switch zerglings with:
echo q | sudo -u galaxy tee /srv/galaxy/var/zerg-old.fifo
# or as the galaxy user:
echo q >/srv/galaxy/var/zerg-old.fifo