社内クラスタのセットアップ

hosts

192.168.201.202	fc1
192.168.201.203	fc2
192.168.201.204	fc3
192.168.201.205	fc4
192.168.201.206	fc5

interface

# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

# The loopback network interface
auto lo
iface lo inet loopback

# The primary network interface
auto eth0
iface eth0 inet static
    address 192.168.201.XXX
    netmask 255.255.255.0
    network 192.168.1.0
    broadcast 192.168.1.255
    gateway 192.168.201.254

setup.sh

#!/bin/bash
# setup environment variables
echo "
JAVA_HOME=/usr/lib/jvm/java-6-sun
export JAVA_HOME
M2_HOME=/usr/local/lib/apache-maven-3.0.3
export M2_HOME

PATH=\$PATH:\$JAVA_HOME/bin:\$M2_HOME/bin
export PATH
" > ~/.marblerc

chmod a+rxw ~/.marblerc

echo "

. .marblerc
" >> ~/.bashrc

# setup ssh
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa 
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
ssh localhost -o StrictHostKeyChecking=no exit

# setup libraries
sudo apt-get -y install curl
curl -s http://archive.cloudera.com/debian/archive.key | sudo apt-key add -
sudo apt-get update
sudo apt-get -y install ssh
sudo apt-get -y install git
sudo apt-get -y install vim
sudo apt-get -y install sun-java6-jdk 
sudo apt-get -y install subversion-tools
sudo apt-get -y install ruby
sudo apt-get -y install ant
sudo apt-get -y install sysv-rc-conf

sudo apt-get -y install hadoop-0.20
sudo apt-get -y install hadoop-0.20-namenode
sudo apt-get -y install hadoop-0.20-datanode
sudo apt-get -y install hadoop-0.20-secondarynamenode
sudo apt-get -y install hadoop-0.20-jobtracker
sudo apt-get -y install hadoop-0.20-tasktracker

sudo apt-get -y install flume
sudo apt-get -y install sqoop
sudo apt-get -y install hadoop-hbase
sudo apt-get -y install hadoop-zookeeper
sudo apt-get -y install oozie
sudo apt-get -y install oozie-client

# install maven3
wget http://ftp.kddilabs.jp/infosystems/apache//maven/binaries/apache-maven-3.0.3-bin.tar.gz
tar xvzf apache-maven-3.0.3-bin.tar.gz
rm apache-maven-3.0.3-bin.tar.gz
sudo mv apache-maven-3.0.3/ /usr/local/lib/

# install vim utilities
wget http://www.vim.org/scripts/download_script.php?src_id=10654
mkdir -p ~/.vim/syntax/
mv download_script.php?src_id=10654 ~/.vim/syntax/pig.vim
echo "
augroup filetypedetect 
  au BufNewFile,BufRead *.pig set filetype=pig syntax=pig 
augroup END
" >> ~/.vimrc

wget wget https://github.com/vim-ruby/vim-ruby/zipball/vim7.3
unzip vim7.3
ruby vim-ruby-vim-ruby-*/bin/vim-ruby-install.rb

手作業でやること

・hostsの書き換え
・dhcpじゃなくてstaticに割り振るあれの書き換え
・source.listについか
/etc/apt/sources.list

# add sun java repository
deb http://archive.canonical.com/ lucid partner

# add cloudera repository
deb http://archive.cloudera.com/debian maverick-cdh3u1 contrib
deb-src http://archive.cloudera.com/debian maverick-cdh3u1 contrib


・setup.shをとってきて実行

mkdir work
cd work
wget https://raw.github.com/gist/1215998/5e42357876a15ac27c213ab4b491d97555b874f1/setup.sh
chmod u+x setup.sh
./setup.sh