How to Storm

gistfile1.txt

/*******************************************************/
/*                                                     */
/*                        NOTE                         */
/*                                                     */
/*******************************************************/

This has been tested on a clean install of Ubuntu 12 server



/*******************************************************/
/*                                                     */
/*               INSTALLING REQUIREMENTS               */
/*                                                     */
/*******************************************************/

sudo apt-get update
apt-get install openjdk-6-jdk make build-essential
apt-get install uuid-dev unzip pkg-config libtool automake


/*******************************************************/
/*                                                     */
/*                  INSTALLING STORM                   */
/*                                                     */
/*******************************************************/

1) Install ZooKeeper
	wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.5/zookeeper-3.4.5.tar.gz
	tar -xvzf zookeeper-3.4.5.tar.gz

2) Configure Zookeeper
	cp ~/zookeeper-3.4.5/conf/zoo_sample.cfg  ~/zookeeper-3.4.5/conf/zoo.cfg 
	vim ~/zookeeper-3.4.5/conf/zoo.cfg 

	Contents should be:
    
		# The number of milliseconds of each tick
		tickTime=2000
		# The number of ticks that the initial 
		# synchronization phase can take
		initLimit=10
		# The number of ticks that can pass between 
		# sending a request and getting an acknowledgement
		syncLimit=5
		# the directory where the snapshot is stored.
		# do not use /tmp for storage, /tmp here is just 
		# example sakes.
		dataDir=/var/zookeeper
		# the port at which the clients will connect
		clientPort=2181
		#
		# Be sure to read the maintenance section of the 
		# administrator guide before turning on autopurge.
		#
		# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
		#
		# The number of snapshots to retain in dataDir
		#autopurge.snapRetainCount=3
		# Purge task interval in hours
		# Set to "0" to disable auto purge feature
		#autopurge.purgeInterval=1
	
    
    Then:
	cd /var/zookeeper
	echo 1 > myId
	cd ~/zookeeper-3.4.5

3) Run Zookeeper (use a new terminal)
	java -cp zookeeper-3.4.5.jar:lib/*:conf org.apache.zookeeper.server.quorum.QuorumPeerMain ./conf/zoo.cfg

4) Install ZeroMQ
	sudo apt-get update
	sudo apt-get install uuid-dev
	wget http://download.zeromq.org/zeromq-2.1.7.tar.gz
	cd zeromq-2.1.7
	./configure
	make
	make install
    sudo ldconfig

5) Install JZMQ
	sudo add-apt-repository ppa:chris-lea/zeromq
	sudo aptitude install libzmq-dev

6) Install Storm
	cd ~
	wget https://github.com/downloads/nathanmarz/storm/storm-0.8.1.zip
	unzip storm-0.8.1.zip
	mkdir /mnt/storm
	cd storm-0.8.1/
	vim conf/storm.yaml and change the following parameters
			storm.zookeeper.servers:
			     - "127.0.0.1"
			nimbus.host: "localhost"
			storm.local.dir: "/mnt/storm"
			supervisor.slots.ports:
			    - 6700
			    - 6701
			    - 6702
			    - 6703


/*******************************************************/
/*                                                     */
/*                  SET UP SUPERVISORD                 */
/*                                                     */
/*******************************************************/

1) Install supervisord
apt-get install supervisor

2) Edit /etc/supervisor/conf.d and add the following lines:

     [program:nimbus]
        command=/home/ubuntu/storm-0.8.1/bin/storm nimbus
        autostart=true
        autorestart=true
        stderr_logfile=/var/log/strom/nimbus.err.log
        stdout_logfile=/var/log/storm/nimbus.out.log

    [program:ui]
        command=home/ubuntu/storm-0.8.1/bin/storm ui
        autostart=true
        autorestart=true
        stderr_logfile=/var/log/strom/ui.err.log
        stdout_logfile=/var/log/storm/ui.out.log
    
    [program:supervisor]
        command=home/ubuntu/storm-0.8.1/bin/storm supervisor
        autostart=true
        autorestart=true
        stderr_logfile=/var/log/strom/nimbus.err.log
        stdout_logfile=/var/log/storm/nimbus.out.log
    

3) Restart supervisord
service supervisor restart

4) See the Storm UI
Browse to http://localhost:8080/ and see the cluster running


/*******************************************************/
/*                                                     */
/*                  RUNNING TOPOLOGY                   */
/*                                                     */
/*******************************************************/

1) Submit and run topology
	~/storm-0.8.1/bin/storm jar path/to/stormtraining-0.0.1-SNAPSHOT-jar-with-dependencies.jar stormtraining.TwitterRollingHashtagsTopology