mesos on hadoop

error1.sh

STARTUP_MSG:   java = 1.8.0_25
************************************************************/
14/10/28 06:27:07 INFO mapred.JobTracker: registered UNIX signal handlers for [TERM, HUP, INT]
14/10/28 06:27:08 FATAL mapred.JobTracker: java.lang.IllegalArgumentException: Does not contain a valid host:port authority: local
	at org.apache.hadoop.net.NetUtils.createSocketAddr(NetUtils.java:211)
	at org.apache.hadoop.net.NetUtils.createSocketAddr(NetUtils.java:163)
	at org.apache.hadoop.net.NetUtils.createSocketAddr(NetUtils.java:152)
	at org.apache.hadoop.mapred.JobTracker.getAddress(JobTracker.java:2165)
	at org.apache.hadoop.mapred.JobTracker.<init>(JobTracker.java:1764)
	at org.apache.hadoop.mapred.JobTracker.<init>(JobTracker.java:1757)
	at org.apache.hadoop.mapred.JobTracker.startTracker(JobTracker.java:305)
	at org.apache.hadoop.mapred.JobTracker.startTracker(JobTracker.java:297)
	at org.apache.hadoop.mapred.JobTracker.main(JobTracker.java:4547)

14/10/28 06:27:08 INFO mapred.JobTracker: SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down JobTracker at ip-10-0-1-244/10.0.1.244
************************************************************/

error2.sh

root@ip-10-0-1-244:/opt/hadoop/bin# vim start-mapred.sh
root@ip-10-0-1-244:/opt/hadoop/bin# ./start-mapred.sh
starting jobtracker, logging to /opt/hadoop-2.5.0-cdh5.2.0/bin-mapreduce1/../logs/hadoop-root-jobtracker-ip-10-0-1-244.out
May not run daemons as root. Please specify HADOOP_JOBTRACKER_USER
root@ip-10-0-1-244:/opt/hadoop/bin# HADOOP_JOBTRACKER_USER=mapred ./start-mapred.sh
starting jobtracker, logging to /opt/hadoop-2.5.0-cdh5.2.0/bin-mapreduce1/../logs/hadoop-root-jobtracker-ip-10-0-1-244.out
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /opt/hadoop-2.5.0-cdh5.2.0/bin-mapreduce1/../logs/hadoop-root-jobtracker-ip-10-0-1-244.log (Permission denied)
	at java.io.FileOutputStream.open(Native Method)
	at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
	at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
	at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
	at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
	at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollingFileAppender.java:223)
	at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
	at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
root@ip-10-0-1-244:/opt/hadoop/bin# ls

provision.sh

#!/bin/bash

pushd /tmp
  wget http://archive.cloudera.com/cdh5/cdh/5/hadoop-2.5.0-cdh5.2.0.tar.gz
  tar -xzf hadoop-2.5.0-cdh5.2.0.tar.gz
  rm hadoop-2.5.0-cdh5.2.0.tar.gz
  
  # download and build framework
  git clone https://github.com/mesos/hadoop.git
  pushd hadoop
    mvn -Dmesos.version=0.20.1 clean package
  popd
  
  # build hadoop-mesos distribution
  cp hadoop/target/hadoop-mesos-0.0.8.jar hadoop-2.5.0-cdh5.2.0/share/hadoop/common/lib/
  pushd hadoop-2.5.0-cdh5.2.0
    mv bin bin-mapreduce2
    mv examples examples-mapreduce2 
    ln -s bin-mapreduce1 bin
    ln -s examples-mapreduce1 examples

    pushd etc
      mv hadoop hadoop-mapreduce2
      ln -s hadoop-mapreduce1 hadoop
    popd

    pushd share/hadoop
      rm mapreduce
      ln -s mapreduce1 mapreduce
    popd
  popd

  # package up hadoop distribution
  tar czf hadoop-2.5.0-cdh5.2.0.tar.gz hadoop-2.5.0-cdh5.2.0
popd

# Upload hadoop distribution to S3 as a public downloadable artifact

# download the hadoop-distribution
pushd /opt
  wget https://s3.amazonaws.com/com.brightcove.rna.repo.dev/hadoop-2.5.0-cdh5.2.0.tar.gz
  tar xzf hadoop-2.5.0-cdh5.2.0.tar.gz && rm *.tar.gz
  ln -s hadoop-2.5.0-cdh5.2.0 hadoop

  # create hadoop config
  cd /opt/hadoop/etc/hadoop
  cat > mapred-site.xml <<EOF
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>mapred.job.tracker</name>
    <value>${HOSTNAME}:10001</value>
  </property>
</configuration>
EOF
cat > hdfs-site.xml <<EOF
<configuration>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
</configuration>
EOF
  cat > core-site.xml <<EOF
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>mapred.job.tracker</name>
    <value>${HOSTNAME}:10001</value>
  </property>
  <property>
    <name>mapred.jobtracker.taskScheduler</name>
    <value>org.apache.hadoop.mapred.MesosScheduler</value>
  </property>
  <property>
    <name>mapred.mesos.taskScheduler</name>
    <value>org.apache.hadoop.mapred.JobQueueTaskScheduler</value>
  </property>
  <property>
    <name>mapred.mesos.master</name>
    <value>$(cat /etc/mesos/zk)</value>
  </property>
  <property>
    <name>mapred.mesos.executor.uri</name>
    <value>https://s3.amazonaws.com/com.brightcove.rna.repo.dev/hadoop-2.5.0-cdh5.2.0.tar.gz</value>
  </property>
</configuration>
EOF
mkdir -p /tmp/hadoop-tmpdir

  echo 'export JAVA_HOME=/usr/lib/jvm/java-8-oracle' >> hadoop-env.sh
  echo 'export MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so' >> hadoop-env.sh
popd