SwiftRiver setup script

swiftriver-setup.sh

#!/bin/sh
#
# Author: Emmanuel Kala <emmauel@ushahidi.com>
# Description: This script deploys SwiftRiver (API and service apps)
# on single host.
#

# Update the sources
# apt-get update

# Update the packages
# apt-get -y upgrade

# First things first, install the firewall
# apt-get -y install ufw

# Enable the firewall and setup basic rules for traffic flow
# ufw enable
# ufw default deny
# ufw allow 22
# ufw allow 80
# ufw allow 9418

# Disable SSH root logins
#  sed -i 's/PermitRootLogin yes/PermitRootLogin no/g' /etc/ssh/sshd_config
#  service ssh restart

# RabbitMQ Server
apt-get -y install rabbitmq-server

# Dependencies for the service stack applications
apt-get -y install python-pip python-mysqldb python-imaging python-lxml
pip install tweepy
pip install pika
pip install feedparser
pip install python-cloudfiles

# Install maven - for building SwiftRiver Java applications
apt-get -y install maven2

# Remove the application directories (if the exist) 
# and un-install the swiftriver service
if [ -f /etc/init.d/swiftriver ];
then
	echo "Removing previous swiftriver installation"
	service swiftriver stop
	update-rc.d -f swiftriver remove
	rm -f /etc/init.d/swiftriver
fi

# Delete SwiftRiver source directories
rm -rf /opt/swiftriver

# Apache Tomcat home directory
TOMCAT_HOME=/opt/tomcat

# Create the directories for SwiftRiver
mkdir -p /opt/swiftriver/src
mkdir -p /opt/swiftriver/api
mkdir -p /opt/swiftriver/services/python
mkdir -p /opt/swiftriver/solr-data/swiftriver

# Clone the repositories for supporting applications
cd /opt/swiftriver/src

#SwiftRiver-API
git clone git://github.com/ushahidi/SwiftRiver-API.git
cd SwiftRiver-API
mvn clean package
cp target/swiftriver-api.war /opt/swiftriver/api
cp target/classes/config/swiftriver-api.xml $TOMCAT_HOME/conf/Catalina/localhost
cp target/classes/indexer.properties /opt/swiftriver/api
perl -p -i -e 's/docBase=\"(.+)\"\s/docBase=\"\/opt\/swiftriver\/api\/swiftriver\.war\" /g' $TOMCAT_HOME/conf/Catalina/localhost/swiftriver-api.xml
perl -p -i -e 's/src.*indexer\.properties/\/opt\/swiftriver\/api\/indexer\.properties/g' $TOMCAT_HOME/conf/Catalina/localhost/swiftriver-api.xml

# Solr
cp solr/solr.xml /opt/swiftriver/solr-data
cp -rf solr/swiftriver/* /opt/swiftriver/solr-data/swiftriver

cd ..

# SwiftRiver-Core
git clone git://github.com/ushahidi/SwiftRiver-Core.git

# Copy the rss, semanticsqueue amd mediaextractor apps to the service directory
cp -rf SwiftRiver-Core/rss /opt/swiftriver/services/python
cp -rf SwiftRiver-Core/twitter /opt/swiftriver/services/python
cp -rf SwiftRiver-Core/semanticsqueue /opt/swiftriver/services/python
cp -rf SwiftRiver-Core/mediaextractor /opt/swiftriver/services/python

# Create the log directories for the python apps
for dir in `ls -d /opt/swiftriver/services/python/*`; do
	if [ -d "$dir/config" ]; then
		mkdir -p $dir/logs
		for file in $dir/config/*.cfg.template
		do
			target=`basename ${file} .cfg.template`.cfg
			cp $file $dir/config/$target
			perl -p -i -e 's/\/path\/to\/logs/${dir}\/logs/g' $dir/config/$target
			perl -p -i -e 's/(\/\w+)+\/(\w+\.pid)/\/tmp\/$2/g' $dir/config/$target
		done
	fi
done

# Copy the lib directory
cp -rf SwiftRiver-Core/lib /opt/swiftriver/services/python

# Build and install the swiftriver-api-client
git clone git://github.com/ushahidi/swiftriver-api-java.git
cd swiftriver-api-java &&  mvn clean install && cd ..

# Package the dropqueue processor
cd /opt/swiftriver/src
git clone git://github.com/ushahidi/swiftriver-core-dropqueue-processor.git dropqueue-processor
cd dropqueue-processor
mvn clean package
cp -rf target/generated-resources/appassembler/jsw/dropqueue-processor /opt/swiftriver/services
cp config/* /opt/swiftriver/services/dropqueue-processor/conf
cd ..

# Package the rules processor
cd /opt/swiftriver/src
git clone git://github.com/ushahidi/swiftriver-core-rules-processor.git rules-processor
cd rules-processor
mvn clean package
cp -rf target/generated-resources/appassembler/jsw/rules-processor /opt/swiftriver/services
cp config/* /opt/swiftriver/services/rules-processor/conf

cd /opt/swiftriver/services
perl -p -i -e 's/-Dext\.prop\.dir/-Dext\.prop\.dir=\/opt\/swiftriver\/services\/dropqueue-processor\/conf/g' dropqueue-processor/conf/wrapper.conf
perl -p -i -e 's/-Dext\.prop\.dir/-Dext\.prop\.dir=\/opt\/swiftriver\/services\/rules-processor\/conf/g' rules-processor/conf/wrapper.conf

# Make the wrappers executable
chmod +x dropqueue-processor/bin/dropqueue-processor
chmod +x dropqueue-processor/bin/wrapper-linux*
chmod +x rules-processor/bin/rules-processor
chmod +x rules-processor/bin/wrapper-linux* 

# Run the remaining ops from the home directory
cd ~/

# Create the swiftriver script
cat <<EOF > swiftriver
#!/bin/bash
### BEGIN INIT INFO
# Provides:	SwiftRiver service stack bootstrap
# Required-Start:
# Required-Stop:
# Default-Start:
# Default-Stop:
# Short-Description: Automatically start the background crawlers for SwiftRiver
# Description
### END INIT INFO

# Set the PYTHONPATH
export PYTHONPATH=$PYTHONPATH:/opt/swiftriver/services/python/lib

# cd into the directory with the services dir
cd /opt/swiftriver/services

start_services() {
	# RSS Scheduler and Fetcher 
	python python/rss/rss_scheduler.py start
	python python/rss/rss_fetcher.py start

	# Semantics queue and media extraction
	python python/semanticsqueue/semanticsqueue.py start
	python python/mediaextractor/mediaextractor.py start
	
	# Twitter
	python python/twitter/manager.py start
	python python/twitter/firehose.py start

	# DropQueue processor and rules processor
	dropqueue-processor/bin/dropqueue-processor start
	rules-processor/bin/rules-processor start
}

# Stops the content services
stop_services() {
	# Twitter
	python python/twitter/firehose.py stop
	python python/twitter/manager.py stop

	# RSS
	python python/rss/rss_fetcher.py stop
	python python/rss/rss_scheduler.py stop

	# Queues
	python python/semanticsqueue/semanticsqueue.py stop
	python python/mediaextractor/mediaextractor.py stop

	# DropQueue and rules processor
	dropqueue-processor/bin/dropqueue-processor stop
	rules-processor/bin/rules-processor stop
}

case \$1 in
	start)
		echo "Starting SwiftRiver content services"
		start_services
	;;
	stop)
		echo "Stopping SwiftRiver content services"
		stop_services
	;;
	restart)
		echo "Restarting SwiftRiver content services"
		stop_services
		start_services
	;;
	*)
		echo "Usage: /etc/init.d/swiftriver (start|stop|restart)"
		exit 1
	;;
esac
EOF

cp swiftriver /etc/init.d
chmod a+x /etc/init.d/swiftriver

# Add the swiftriver service
update-rc.d swiftriver defaults 95 05

echo
echo "SwiftRiver installation completed!"
echo

exit