Skip to content

Instantly share code, notes, and snippets.

@eendeego
Last active December 11, 2015 01:08
Show Gist options
  • Save eendeego/4520827 to your computer and use it in GitHub Desktop.
Save eendeego/4520827 to your computer and use it in GitHub Desktop.
mkdir hadoop
cd hadoop
wget http://mirrors.fe.up.pt/pub/apache/hadoop/common/hadoop-1.1.1/hadoop-1.1.1-bin.tar.gz
tar zxvf hadoop-1.1.1-bin.tar.gz
mkdir conf
cp hadoop-1.1.1/conf/* conf/
export HADOOP_CONF_DIR=$PWD/conf
export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.7.0_12.jdk/Contents/Home
# This should work
export HADOOP_OPTS="-Djava.security.krb5.realm= -Djava.security.krb5.kdc="
# This actually works
export HADOOP_OPTS="-Djava.security.krb5.realm=OX.AC.UK -Djava.security.krb5.kdc=kdc0.ox.ac.uk:kdc1.ox.ac.uk"
perl -pi -e "s|# export JAVA_HOME=.*|\${_}export JAVA_HOME=$JAVA_HOME|g" conf/hadoop-env.sh
perl -pi -e "s|# export HADOOP_OPTS=.*|\${_}export HADOOP_OPTS=\"$HADOOP_OPTS\"|g" conf/hadoop-env.sh
sed -i '' '/<configuration>/a\
\ <property>\
\ <name>fs.default.name</name>\
\ <value>hdfs://localhost:9000</value>\
\ </property>' conf/core-site.xml
sed -i '' '/<configuration>/a\
\ <property>\
\ <name>dfs.replication</name>\
\ <value>1</value>\
\ </property>' conf/hdfs-site.xml
sed -i '' '/<configuration>/a\
\ <property>\
\ <name>mapred.job.tracker</name>\
\ <value>localhost:9001</value>\
\ </property>' conf/mapred-site.xml
ssh-keygen -t dsa -P '' -f ~/.ssh/localhost
cat ~/.ssh/localhost.pub >> ~/.ssh/authorized_keys
export PATH=hadoop-1.1.1/bin:$PATH
hadoop version
hadoop jar hadoop-1.1.1/hadoop-examples-1.1.1.jar pi 10 100
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment