Setting up a Vagrant cluster for testing out Apache Hadoop.

base-hadoop.pp

include hadoop

group { "puppet":
  ensure => "present",
}

exec { "apt-get update":
  command => "apt-get update",
     path => "/usr/bin",
}

package { "openjdk-6-jdk":
   ensure => present,
  require => Exec["apt-get update"],
}

file { "/root/.ssh":
  ensure => "directory",
  require => Exec['apt-get update']
}

file { "/root/.ssh/id_rsa":
  source => "puppet:///modules/hadoop/id_rsa",
    mode => 600,
   owner => root,
   group => root,
 require => Exec["apt-get update"],
}

file { "/root/.ssh/id_rsa.pub":
  source => "puppet:///modules/hadoop/id_rsa.pub",
    mode => 644,
   owner => root,
   group => root,
 require => Exec["apt-get update"],
}

ssh_authorized_key { "ssh_key":
  ensure  => "present",
  key     => "AAAAB3NzaC1yc2EAAAADAQABAAABAQDEFC7jZNwXhqwdms4AGbyDASGplPPlsMnn6Sidrv4pUY8A5R0DGJCweKMh3cel+W1r3iczZCreV80MXVbvTCeCgrlignC+qaxgZ/RYIe1e9w5wjGEhAW2YRrz4cIntXiD79FdOWCe7O/VsNk4piFPDVG2JQztjP+6nREvGDDB6wa5BHQvr/9kRg16eutrmlNSeaPvTRKF5EFOq1dCqhedV17LxS0gSDgd1v/fOhlm2JNCI5CaaGxS3MPn1eMTA1Mf3W1X9y8w01oRUxiojnXw6zhWXHSDQiJliZkV0wovHkCOd3bdBNiusQara9sdLc5I/kkT8UbEBU7uevNSEH0/R",
  type    => "ssh-rsa",
  user    => "root",
  require => File["/root/.ssh/id_rsa.pub"],
}

core-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>fs.default.name</name>
    <value>hdfs://master:9000</value>
    <description>The name of the default file system. A URI whose scheme and authority determine the FileSystem implementation.</description>
  </property>
</configuration>

hadoop-env.sh

# Set Hadoop-specific environment variables here.

# The only required environment variable is JAVA_HOME.  All others are
# optional.  When running a distributed configuration it is best to
# set JAVA_HOME in this file, so that it is correctly defined on
# remote nodes.

# The java implementation to use.  Required.
export JAVA_HOME=/usr/lib/jvm/java-6-openjdk

# export HADOOP_CLASSPATH=

# The maximum amount of heap to use, in MB. Default is 1000.
# export HADOOP_HEAPSIZE=2000

# Extra Java runtime options.  Empty by default.
# export HADOOP_OPTS=-server

# Command specific options appended to HADOOP_OPTS when specified
export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_NAMENODE_OPTS"
export HADOOP_SECONDARYNAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_SECONDARYNAMENODE_OPTS"
export HADOOP_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_DATANODE_OPTS"
export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPTS"
export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS"
# export HADOOP_TASKTRACKER_OPTS=
# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
# export HADOOP_CLIENT_OPTS

# Extra ssh options.  Empty by default.
# export HADOOP_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR"

# Where log files are stored.  $HADOOP_HOME/logs by default.
# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs

# File naming remote slave hosts.  $HADOOP_HOME/conf/slaves by default.
# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves

# host:path where hadoop code should be rsync'd from.  Unset by default.
# export HADOOP_MASTER=master:/home/$USER/src/hadoop

# Seconds to sleep between slave commands.  Unset by default.  This
# can be useful in large clusters, where, e.g., slave rsyncs can
# otherwise arrive faster than the master can service them.
# export HADOOP_SLAVE_SLEEP=0.1

# The directory where pid files are stored. /tmp by default.
# export HADOOP_PID_DIR=/var/hadoop/pids

# A string representing this instance of hadoop. $USER by default.
# export HADOOP_IDENT_STRING=$USER

# The scheduling priority for daemon processes.  See 'man nice'.
# export HADOOP_NICENESS=10

hdfs-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>dfs.replication</name>
    <value>3</value>
    <description>The actual number of replications can be specified when the file is created.</description>
  </property>
</configuration>

init.pp

class hadoop {
  $hadoop_home = "/opt/hadoop-1.0.4"

  exec { "download_hadoop":
    command => "wget -O /tmp/hadoop.tar.gz http://apache.mirrors.pair.com/hadoop/common/hadoop-1.0.4/hadoop-1.0.4.tar.gz",
    path => $path,
    unless => "ls /opt | grep hadoop-1.0.4",
    require => Package["openjdk-6-jdk"],
  }

  exec { "unpack_hadoop" :
    command => "tar xzf /tmp/hadoop.tar.gz -C /opt",
    path => $path,
    creates => "$hadoop_home",
    require => Exec["download_hadoop"],
  }

  file { "$hadoop_home/conf/masters":
     source => "puppet:///modules/hadoop/masters",
       mode => 644,
      owner => root,
      group => root,
    require => Exec["unpack_hadoop"]
  }

  file { "$hadoop_home/conf/slaves":
     source => "puppet:///modules/hadoop/slaves",
       mode => 644,
      owner => root,
      group => root,
    require => Exec["unpack_hadoop"]
  }

  file { "$hadoop_home/conf/core-site.xml":
     source => "puppet:///modules/hadoop/core-site.xml",
       mode => 644,
      owner => root,
      group => root,
    require => Exec["unpack_hadoop"]
  }

  file { "$hadoop_home/conf/hdfs-site.xml":
     source => "puppet:///modules/hadoop/hdfs-site.xml",
       mode => 644,
      owner => root,
      group => root,
    require => Exec["unpack_hadoop"]
  }

  file { "$hadoop_home/conf/mapred-site.xml":
     source => "puppet:///modules/hadoop/mapred-site.xml",
       mode => 644,
      owner => root,
      group => root,
    require => Exec["unpack_hadoop"]
  }

  file { "$hadoop_home/conf/hadoop-env.sh":
     source  => "puppet:///modules/hadoop/hadoop-env.sh",
       mode  => 644,
      owner  => root,
      group  => root,
     require => Exec["unpack_hadoop"],
  }
}

mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>mapred.job.tracker</name>
    <value>master:9001</value>
    <description>The host and port that the MapReduce job tracker runs at.</description>
  </property>
</configuration>

masters

192.168.2.10

slaves

192.168.2.12 192.168.2.13 192.168.2.14

Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
  config.vm.box = "base-hadoop"
  config.vm.box_url = "http://files.vagrantup.com/lucid64.box"

  config.vm.provision :puppet do |puppet|
    puppet.manifests_path = "manifests"
    puppet.manifest_file  = "base-hadoop.pp"
    puppet.module_path    = "modules"
  end

  config.vm.define :master do |master_config|
    master_config.vm.network :private_network, ip: "192.168.2.10"
    master_config.vm.hostname = "master"
  end

  config.vm.define :backup do |backup_config|
    backup_config.vm.network :private_network, ip: "192.168.2.11"
    backup_config.vm.hostname = "backup"
  end

  config.vm.define :hadoop1 do |hadoop1_config|
    hadoop1_config.vm.network :private_network, ip: "192.168.2.12"
    master_config.vm.hostname = "hadoop1"
  end

  config.vm.define :hadoop2 do |hadoop2_config|
    hadoop2_config.vm.network :private_network, ip: "192.168.2.13"
    master_config.vm.hostname = "hadoop2"
  end

  config.vm.define :hadoop3 do |hadoop3_config|
    hadoop3_config.vm.network :private_network, ip: "192.168.2.14"
    master_config.vm.hostname = "hadoop3"
  end

end