Gitorious installation script for Ubuntu 12.04 LTS

install.sh

#!/bin/bash
# You should clone this gist to obtain the installation and patch file
# and then run it locally, after changing the below settings to something
# suitable for your system. Something like this:
#   sudo apt-get install -y ssh
#   cd
#   scp -r nhaigh@bioserver:git/sysadmin/generic/gitorious_install ./
# Edit the settings below, then run the following as root:
#   cd gitorious_install && sh ./install.sh
#
#------------------------------------------------------------------------------
# HELPFUL ONLINE DOCUMENTATION
#------------------------------------------------------------------------------
# https://gitorious.org/gitorious/pages/UbuntuInstallation
# https://gitorious.org/gitorious/mainline/source/03c30468ccf79f3cde127c042dff59c9a63ccfe0:doc/setup-dev-env-ubuntu.sh#L72-181
# http://blog.gitorious.org/2013/01/04/gitorious-3-0-lands-in-the-next-branch/
# https://gitorious.org/gitorious/mainline/source/39de776f3988b8f7fc98085aaacae528eef39fe0:doc/upgrading-to-gitorious3.md
# http://edin.no-ip.com/blog/hswong3i/gitorious-ubuntu-12-04-mini-howto
# https://github.com/schacon/git-sphinx
#------------------------------------------------------------------------------

#------------------------------------------------------------------------------
# SETTINGS
#------------------------------------------------------------------------------
source ~/gitorious_install/gitorious_settings.txt

MYSQL_ROOT_PASSWORD=`< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-15};echo;`
MYSQL_GITORIOUS_USER=gitorious
MYSQL_GITORIOUS_PASSWORD=`< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-15};echo;`
GITORIOUS_HOST=`hostname -f`
GITORIOUS_DIR=/var/www/gitorious
# SYSADMIN details i.e. the user running this script
SYSADMIN=$USER
SYSADMIN_EMAIL=${SYSADMIN}@$GITORIOUS_HOST
# gitorious user details
GITORIOUS_USER='git'
#GITORIOUS_USER_PASSWORD=`< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-15};echo;`
# gitorious admin details
GITORIOUS_ADMIN=$USER
GITORIOUS_ADMIN_EMAIL=${GITORIOUS_ADMIN}@${GITORIOUS_HOST}
GITORIOUS_ADMIN_PASSWORD=`< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c${1:-15};echo;`

cat > ~/gitorious_settings.txt <<__SCRIPT__
################################
# Your Gitorious install details
# You can source this file to reset important variables
################################
MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD}
MYSQL_GITORIOUS_USER=${MYSQL_GITORIOUS_USER}
MYSQL_GITORIOUS_PASSWORD=${MYSQL_GITORIOUS_PASSWORD}
GITORIOUS_DIR=${GITORIOUS_DIR}
GITORIOUS_USER=${GITORIOUS_USER}
GITORIOUS_ADMIN_EMAIL=${GITORIOUS_ADMIN_EMAIL}
GITORIOUS_ADMIN_PASSWORD=${GITORIOUS_ADMIN_PASSWORD}

#  MySQL passwords:
#    root:${MYSQL_ROOT_PASSWORD}
#    ${MYSQL_GITORIOUS_USER}:${MYSQL_GITORIOUS_PASSWORD}
#  Gitorious Dir:
#    ${GITORIOUS_DIR}
#  Gitorious User:
#    ${GITORIOUS_USER}
#  Gitorious Admin:
#    ${GITORIOUS_ADMIN_EMAIL}:${GITORIOUS_ADMIN_PASSWORD}
################################
__SCRIPT__

#------------------------------------------------------------------------------
# INSTALLATION STARTS HERE
#------------------------------------------------------------------------------

apt-get update && apt-get dist-upgrade -y
# install some useful packages
apt-get install -y colordiff locate ntp pwgen rsync vim

# install postfix for sending out emails to users etc
echo "postfix postfix/main_mailer_type select Internet Site" | debconf-set-selections
echo "postfix postfix/mailname string ${HOSTNAME}" | debconf-set-selections
apt-get install -y postfix

# install MySQL 5.5
echo "mysql-server-5.5 mysql-server/root_password password $MYSQL_ROOT_PASSWORD" | debconf-set-selections
echo "mysql-server-5.5 mysql-server/root_password_again password $MYSQL_ROOT_PASSWORD" | debconf-set-selections
echo "mysql-server-5.5 mysql-server/root_password seen true" | debconf-set-selections
echo "mysql-server-5.5 mysql-server/root_password_again seen true" | debconf-set-selections
apt-get install -y mysql-server mysql-client

# Install the apache2 webserver and enable module Rewrite
apt-get install -y apache2 memcached
a2enmod rewrite && service apache2 restart

# Install Ruby, Interactive Ruby, SSL support for Ruby, and update system Gems
apt-get install -y \
  ruby1.9.1 ruby1.9.1-dev rdoc1.9.1 \
  irb1.9.1 ri1.9.1 \
  libopenssl-ruby1.9.1 libssl-dev \
  build-essential
REALLY_GEM_UPDATE_SYSTEM=1 gem update --system
gem install -b --no-ri --no-rdoc bundler

# Fetch Gitorious
#   See what rake tasks can be done: bundle exec ${GITORIOUS_DIR}/bin/rake --tasks
apt-get install -y git
git clone git://gitorious.org/gitorious/mainline.git ${GITORIOUS_DIR}
cd ${GITORIOUS_DIR}
git submodule update --init --recursive
# Fix a bug in the Gemfile syntax
sed -i.old -e 's/\(git\|branch\|submodules\): /:\1 => /g' Gemfile
ln -s ${GITORIOUS_DIR}/bin/gitorious /usr/bin

# Download and install the required Gems using bundler but first, lets install some package dependencies
apt-get install -y libxml2-dev libxslt1-dev libicu-dev libmysqlclient-dev libpq-dev
# check we have bundle >= 1.3.5 before we continue
bundle -v
bundle --deployment
#bundle pack --all
#bundle install

# Setup the web/application server side of things
#####
# Install the Ruby application server, called passenger, and install the passenger module for apache2 from the phusionpassenger repo
gem install -b --no-ri --no-rdoc passenger
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 561F9B9CAC40B2F7
cat > /etc/apt/sources.list.d/passenger.list << __PASSENGER__
deb https://oss-binaries.phusionpassenger.com/apt/passenger precise main
__PASSENGER__
chmod 600 /etc/apt/sources.list.d/passenger.list
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 561F9B9CAC40B2F7
apt-get update
apt-get install -y libapache2-mod-passenger

# Create apache2 site config file for gitorious
cat > /etc/apache2/sites-available/gitorious << __GITORIOUS_SITE__
<VirtualHost *:80>
    ServerName `hostname --all-fqdns`
  DocumentRoot ${GITORIOUS_DIR}/public
</VirtualHost>
__GITORIOUS_SITE__

# Create the SSL version of gitorious
a2enmod ssl
cat > /etc/apache2/sites-available/gitorious-ssl << __GITORIOUS_SSL_SITE__
<IfModule mod_ssl.c>
    <VirtualHost _default_:443>
        DocumentRoot ${GITORIOUS_DIR}/public
        SSLEngine on
        SSLCertificateFile    /etc/ssl/certs/ssl-cert-snakeoil.pem
        SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key
        BrowserMatch ".*MSIE.*" nokeepalive ssl-unclean-shutdown downgrade-1.0 force-response-1.0
    </VirtualHost>
</IfModule>
__GITORIOUS_SSL_SITE__

# Configure Gitorious and setup the relevant files/directories/user
cp ~/gitorious_install/config/gitorious.yml config/
mkdir -p /var/git/{repositories,tarballs-work,tarballs}
adduser --system --home ${GITORIOUS_DIR} --no-create-home --group --shell /bin/bash ${GITORIOUS_USER}    
mkdir ${GITORIOUS_DIR}/.ssh
touch ${GITORIOUS_DIR}/.ssh/authorized_keys
chmod 700 ${GITORIOUS_DIR}/.ssh
chmod 600 ${GITORIOUS_DIR}/.ssh/authorized_keys

# copy the backup repositories into /var/git/repositories
tar -xf ~/gitorious_install/gitorious_snapshot.tar --directory /tmp && mv -r /tmp/tmp-backup-workdir/repos/* /var/git/repositories/
# ensure ${GITORIOUS_USER} is the owner of everything under /var/git
chown -R ${GITORIOUS_USER}:${GITORIOUS_USER} /var/git

# Setup the MySQL databases
#####
cp ~/gitorious_install/config/database.yml config/
# Create the gitorious MySQL Database and grant access to it by the user MySQL gitorious user
mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "CREATE DATABASE gitorious"
mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "CREATE DATABASE gitorious_dev"
mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "CREATE DATABASE gitorious_test"
mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "CREATE USER '${MYSQL_GITORIOUS_USER}'@'localhost' IDENTIFIED BY '$MYSQL_GITORIOUS_PASSWORD'"
mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "GRANT ALL PRIVILEGES ON gitorious.* TO '${MYSQL_GITORIOUS_USER}'@'localhost'"
mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "GRANT ALL PRIVILEGES ON gitorious_dev.* TO '${MYSQL_GITORIOUS_USER}'@'localhost'"
mysql -u root --password=${MYSQL_ROOT_PASSWORD} -e "GRANT ALL PRIVILEGES ON gitorious_test.* TO '${MYSQL_GITORIOUS_USER}'@'localhost'"
# Import an existing database generated from mysqldump
mysql -u ${MYSQL_GITORIOUS_USER} --password=$MYSQL_GITORIOUS_PASSWORD gitorious < ~/gitorious_install/gitorious_db.dump
mysql -u ${MYSQL_GITORIOUS_USER} --password=$MYSQL_GITORIOUS_PASSWORD gitorious_dev < ~/gitorious_install/gitorious_db.dump

# ensure ${GITORIOUS_USER} is the owner of the everything under ${GITORIOUS_DIR}
chown -R ${GITORIOUS_USER}:${GITORIOUS_USER} ${GITORIOUS_DIR}

# Migrate the imported data to the DB schema of the current Gitorious Install and Precompile all assets
bin/rake db:migrate
bin/rake assets:precompile

# Disable the default sites and enable the Gitorious sites
a2dissite default && a2dissite default-ssl && a2ensite gitorious && a2ensite gitorious-ssl && service apache2 reload
touch tmp/restart.txt

a2ensite default && a2ensite default-ssl && a2dissite gitorious && a2dissite gitorious-ssl && service apache2 reload
# apache2ctl start

touch tmp/restart.txt

# Setup all the services
######
# Install the Redis key-value server and enable memory over commit for it
apt-get install -y redis-server
cat >> /etc/sysctl.conf << __REDIS_CONFIG__
vm.overcommit_memory=1
__REDIS_CONFIG__
service redis-server restart
# Install Sphinx Search Engine
SPHINXSEARCH_VERSION='2.1.4'
apt-get install -y libodbc1 unixodbc libltdl7 odbcinst1debian2 odbcinst
wget --directory-prefix ~/ -c http://sphinxsearch.com/files/sphinxsearch_${SPHINXSEARCH_VERSION}-release-0ubuntu11~precise_amd64.deb
dpkg -i ~/sphinxsearch_${SPHINXSEARCH_VERSION}-release-0ubuntu11~precise_amd64.deb

# Ensure everything comes up when we reboot. We'll use upstart config files supplied as templates for this
#####
# git-daemon and git-proxy - Add them to the init directory and let upstart know about them
sed -i.old -e 's/\/var\/www\/gitorious\/repositories/\/var\/git\/repositories/' ${GITORIOUS_DIR}/doc/templates/upstart/git-daemons.conf
sed -i.old -e 's/\/app//' ${GITORIOUS_DIR}/doc/templates/upstart/git-proxy.conf
ln -s ${GITORIOUS_DIR}/doc/templates/upstart/git-daemons.conf /etc/init/
ln -s ${GITORIOUS_DIR}/doc/templates/upstart/git-proxy.conf /etc/init/
initctl reload-configuration
start git-daemons

# Redis key-value server and resque-worker
update-rc.d redis-server defaults # Maybe change to upstart?
sed -i.old -e 's/\/app//' ${GITORIOUS_DIR}/doc/templates/upstart/resque-worker.conf
ln -s ${GITORIOUS_DIR}/doc/templates/upstart/resque-worker.conf /etc/init/
initctl reload-configuration
start resque-worker

# Thinking Sphinx search engine
# rebuild search index
bin/rake ts:configure
${GITORIOUS_DIR}/bin/rake ts:index

cat > ${GITORIOUS_DIR}/doc/templates/upstart/thinking-sphinx.conf <<__THINKING_SPHINX__
description "Sphinx Upstart script"

start on startup
stop on shutdown

console output

respawn

script
    env RAILS_ENV=production

    exec sudo -u git -i /usr/bin/searchd --config ${GITORIOUS_DIR}/config/production.sphinx.conf --nodetach
end script
__THINKING_SPHINX__
ln -s ${GITORIOUS_DIR}/doc/templates/upstart/thinking-sphinx.conf /etc/init/
initctl reload-configuration
start thinking-sphinx

# Add a cron job to rebuild sphinx indexes evey 30mins between 06:00-21:00 Mon-Fri
#crontab -u ${GITORIOUS_USER} -e
cat >> /var/spool/cron/crontabs/${GITORIOUS_USER} <<__SPHINX_REBUILD_CRONTAB__
*/30 6-21 * * 1-5 ${GITORIOUS_DIR}/bin/rake ts:rebuild > /dev/null
__SPHINX_REBUILD_CRONTAB__
chown ${GITORIOUS_USER}:crontab /var/spool/cron/crontabs/${GITORIOUS_USER}
chmod 600 /var/spool/cron/crontabs/${GITORIOUS_USER}

# Configure Gitorious for hosts without a valid SSL certificate
cat >> ${GITORIOUS_DIR}/config/smtp.yml << __GITORIOUS_SMTP_CONFIG__
address: localhost
port: 25
enable_starttls_auto: false
openssl_verify_mode: none
__GITORIOUS_SMTP_CONFIG__
touch tmp/restart.txt

#####
# FINAL CHECKS
#####
# Check for certain that all relevant directories/files are owned by ${GITORIOUS_USER}:${GITORIOUS_USER}
chown -R ${GITORIOUS_USER}:${GITORIOUS_USER} ${GITORIOUS_DIR} /var/git
# Ensure we have a symlink to the ${GITORIOUS_DIR}/data/hooks directory in the top of the repositories directory
# TODO Maybe use script/fixup_hooks script?
ln -s ${GITORIOUS_DIR}/data/hooks /var/git/repositories/.hooks
# Restore the .ssh/authorized_keys from backup if necessary otherwise all keys will need to be added back again via the UI
su -c "scp nhaigh@bioserver:git/sysadmin/generic/gitorious_install/authorized_keys ${GITORIOUS_DIR}/.ssh/authorized_keys" ${GITORIOUS_USER}
# Run tests
su - -c "${GITORIOUS_DIR}/bin/rake test:micros" ${GITORIOUS_USER}
touch tmp/restart.txt

#####
# TODO Log rotate
# TODO Check apache2 configuration for tarball generation/downloads in gitorious 
#####

# sudo apt-get install -y build-essential zlib1g-dev libcurl4-openssl-dev apache2 apg geoip-bin libgeoip1 libgeoip-dev libpcre3 libpcre3-dev zlib1g zlib1g-dev libyaml-dev apache2-dev libonig-dev zip unzip memcached git-core git-svn git-doc git-cvs libreadline-dev sqlite3 libsqlite3-dev libmagick++4 libmagick++-dev libapache2-mod-xsendfile libxslt-dev libxml2-dev uuid uuid-dev imagemagick postfix
#sudo a2enmod deflate
#sudo bash -c "cat > gitorious" << EOF
#<VirtualHost *:80>
#    <Directory /var/www/gitorious/public>
#        Options FollowSymLinks
#        AllowOverride None
#        Order allow,deny
#        Allow from All
#    </Directory>
#
#    DocumentRoot /var/www/gitorious/public
#    ServerName $GITORIOUS_HOST
#
#    # Gzip/Deflate
#    # http://fluxura.com/2006/5/19/apache-for-static-and-mongrel-for-rails-with-mod_deflate-and-capistrano-support
#    AddOutputFilterByType DEFLATE text/html text/plain text/xml text/javascript text/css application/x-javascript
#    BrowserMatch ^Mozilla/4 gzip-only-text/html
#    BrowserMatch ^Mozilla/4\.0[678] no-gzip
#    BrowserMatch \bMSIE !no-gzip !gzip-only-text/html
#
#    # Far future expires date
#    <FilesMatch "\.(ico|pdf|flv|jpg|jpeg|png|gif|js|css|swf)$">
#        ExpiresActive On
#        ExpiresDefault "access plus 1 year"
#    </FilesMatch>
#
#    # No Etags
#    FileETag None
#
#    RewriteEngine On
#
#    # Check for maintenance file and redirect all requests
#    RewriteCond %{DOCUMENT_ROOT}/system/maintenance.html -f
#    RewriteCond %{SCRIPT_FILENAME} !maintenance.html
#    RewriteRule ^.*$ /system/maintenance.html [L]
#</VirtualHost>
#EOF