Vagrant to provision a Storm cluster in VirtualBox

java_exe.patch

--- bin/storm~	2013-12-05 10:13:25.000000000 -0800
+++ bin/storm	2014-04-17 20:26:41.011354269 -0700
@@ -30,6 +30,11 @@
 CONFIG_OPTS = []
 CONFFILE = ""
 JAR_JVM_OPTS = os.getenv('STORM_JAR_JVM_OPTS', '')
+JAVA_HOME = os.getenv('JAVA_HOME')
+if JAVA_HOME:
+    JAVA_EXE = os.path.join(JAVA_HOME, 'bin', 'java')
+else:
+    JAVA_EXE = 'java'  # good luck
 
 
 def get_config_opts():
@@ -60,7 +65,7 @@
 def confvalue(name, extrapaths):
     global CONFFILE
     command = [
-        "java", "-client", get_config_opts(), "-Dstorm.conf.file=" + CONFFILE, "-cp", get_classpath(extrapaths), "backtype.storm.command.config_value", name
+        JAVA_EXE, "-client", get_config_opts(), "-Dstorm.conf.file=" + CONFFILE, "-cp", get_classpath(extrapaths), "backtype.storm.command.config_value", name
     ]
     p = sub.Popen(command, stdout=sub.PIPE)
     output, errors = p.communicate()
@@ -113,7 +118,7 @@
 def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[], fork=False):
     global CONFFILE
     all_args = [
-        "java", jvmtype, get_config_opts(),
+        JAVA_EXE, jvmtype, get_config_opts(),
         "-Dstorm.home=" + STORM_DIR, 
         "-Djava.library.path=" + confvalue("java.library.path", extrajars),
         "-Dstorm.conf.file=" + CONFFILE,
@@ -121,9 +126,9 @@
     ] + jvmopts + [klass] + list(args)
     print "Running: " + " ".join(all_args)
     if fork:
-        os.spawnvp(os.P_WAIT, "java", all_args)
+        os.spawnvp(os.P_WAIT, JAVA_EXE, all_args)
     else:
-        os.execvp("java", all_args) # replaces the current process and never returns
+        os.execvp(JAVA_EXE, all_args) # replaces the current process and never returns
 
 def jar(jarfile, klass, *args):
     """Syntax: [storm jar topology-jar-path class ...]

storm-nimbus.conf

[program:storm-nimbus]
user=vagrant
directory=/home/vagrant/storm
environment=JAVA_HOME=@JAVA_HOME@
command=/home/vagrant/storm/bin/storm nimbus
numprocs=1
stdout_logfile=/var/lib/storm/nimbus.out.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10000
stderr_logfile=/var/lib/storm/nimbus.err.log
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10000
autostart=true
autorestart=unexpected
startsecs=10

storm-super.conf

[program:storm-supervisor]
user=vagrant
directory=/home/vagrant/storm
environment=JAVA_HOME=@JAVA_HOME@
command=/home/vagrant/storm/bin/storm supervisor
numprocs=1
stdout_logfile=/var/lib/storm/supervisor.out.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10000
stderr_logfile=/var/lib/storm/supervisor.err.log
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10000
autostart=true
autorestart=unexpected
startsecs=10

storm-ui.conf

[program:storm-ui]
user=vagrant
directory=/home/vagrant/storm
environment=JAVA_HOME=@JAVA_HOME@
command=/home/vagrant/storm/bin/storm ui
numprocs=1
stdout_logfile=/var/lib/storm/ui.out.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10000
stderr_logfile=/var/lib/storm/ui.err.log
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10000
autostart=true
autorestart=unexpected
startsecs=10

storm.yaml

storm.zookeeper.servers:
  - "zoo1"
  - "zoo2"
  - "zoo3"
storm.local.dir: /var/lib/storm

nimbus.host: "zoo1"

Vagrantfile

# -*- mode: ruby -*-
# vi: set ts=2 sts=2 sw=2 expandtab ft=ruby :

# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = '2'

$JVM =<<JVM
set -x
mkdir -p /usr/lib/jvm
cd /usr/lib/jvm
tar xzf /vagrant/jdk-*.tar.gz
chown -R 0:0 jdk*
cd /usr/lib/jvm/jdk*
cat >/etc/profile.d/java.sh<<JJ
JAVA_HOME=$PWD
PATH=\\$JAVA_HOME/bin:\\$PATH
JJ
JVM
$ZOO=<<ZOO
set -x
if [ -f /vagrant/var_cache_apt.tar ]; then
  tar xf /vagrant/var_cache_apt.tar -C /
fi
apt-get update
apt-get install -y build-essential ntp supervisor

cd ~vagrant

tar xzf /vagrant/zookeeper-*.tar.gz
cat >zoo.cfg<<ZCFG
tickTime=2000
dataDir=/var/lib/zookeeper
clientPort=2181
initLimit=5
syncLimit=2
server.1=zoo1:2888:3888
server.2=zoo2:2888:3888
server.3=zoo3:2888:3888
ZCFG
mv -i zoo.cfg ./zookeeper-*/conf
chown -R vagrant:vagrant zookeeper-*
su -c "ln -s zookeeper-* zookeeper" vagrant

mkdir /var/lib/zookeeper
echo `hostname` | sed -e s/zoo// > /var/lib/zookeeper/myid
chown -R vagrant:vagrant /var/lib/zookeeper

## do this so JAVA_HOME loads into the current environment
. /etc/profile.d/java.sh

cat > /etc/supervisor/conf.d/zook.conf<<SUPER
[program:zookeeper]
user=vagrant
directory=/var/lib/zookeeper
environment=JAVA_HOME=$JAVA_HOME
command=/home/vagrant/zookeeper/bin/zkServer.sh start-foreground
numprocs=1
stdout_logfile=/var/lib/zookeeper/daemon.out.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10000
stderr_logfile=/var/lib/zookeeper/daemon.err.log
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10000
autostart=true
autorestart=unexpected
startsecs=10
SUPER

supervisorctl reload

cat >/etc/cron.daily/zookeeper<<CRON1
#! /bin/sh
/home/vagrant/zookeeper/bin/zkCleanup.sh /var/lib/zookeeper/snaps -n 3
CRON1
chmod 755 /etc/cron.daily/zookeeper
ZOO

$STORM =<<STORM
set -x
mkdir /var/lib/storm
chown vagrant:vagrant /var/lib/storm
cd ~vagrant

tar xf /vagrant/storm-*.tar.gz
# stupid storm tar has "." in it
chown vagrant:vagrant .

su -c "ln -s storm-* storm" vagrant
if [ -f /vagrant/storm.yaml ]; then
  cp /vagrant/storm.yaml storm/conf/storm.yaml
fi
if [ -f /vagrant/java_exe.patch ]; then
  cd storm
  patch -p0 < /vagrant/java_exe.patch
  cd ..
fi
chown -R vagrant:vagrant storm-*
if [ -f /vagrant/storm-super.conf ]; then
  cp /vagrant/storm-super.conf /etc/supervisor/conf.d
fi
if [ "zoo1" == "`hostname`" ]; then
  if [ -f /vagrant/storm-nimbus.conf ]; then
    cp /vagrant/storm-nimbus.conf /etc/supervisor/conf.d
  fi
  if [ -f /vagrant/storm-ui.conf ]; then
    cp /vagrant/storm-ui.conf /etc/supervisor/conf.d
  fi
fi

supervisorctl reload
STORM

Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  config.vm.box = 'ubuntu1310_64'
  hostname_map = {
      'zoo1' => '192.168.56.101',
      'zoo2' => '192.168.56.102',
      'zoo3' => '192.168.56.103',
  }
  hosts_txt = hostname_map.map {|hn,ip| "#{ip} #{hn}"}.join("\n")
  $HOSTS =<<HOSTS
sed -i"" -e /`hostname`/d /etc/hosts
cat >>/etc/hosts<<HH
#{hosts_txt}
HH
HOSTS

  # this enumerates the zookeeper hosts specifically
  # because hostname_map might (and likely will) contain
  # storm workers, too, which we might not necessarily
  # want to put zookeeper on them
  zoo_hosts = ['zoo1', 'zoo2', 'zoo3']
  zoo_hosts.each {|h|
      config.vm.define h do |c|
        c.vm.hostname = h
        c.vm.network :private_network, ip: hostname_map[c.vm.hostname]
        c.vm.provider :virtualbox do |vb|
          vb.customize ['modifyvm', :id, '--memory', '512']
        end
        c.vm.provision :shell, :inline => $HOSTS
        c.vm.provision :shell, :inline => $JVM
        c.vm.provision :shell, :inline => $ZOO
        c.vm.provision :shell, :inline => $STORM
      end
  }
end

This needs to be updated to build jzmq otherwise one gets UnsatisifedLinkErrors when trying to run a topology. Hurray for native libraries :-(