Skip to content

Instantly share code, notes, and snippets.

@peterklipfel
Last active November 24, 2015 19:38
Show Gist options
  • Save peterklipfel/7b2e77c795f43e59b4e6 to your computer and use it in GitHub Desktop.
Save peterklipfel/7b2e77c795f43e59b4e6 to your computer and use it in GitHub Desktop.
install spark ubuntu 14.04
sudo apt-get update
sudo apt-get install -y openjdk-7-jdk
sudo su -c 'echo "JAVA_HOME=\"/usr/lib/jvm/java-7-openjdk-amd64\"" >> /etc/environment'
cd /opt
sudo wget http://d3kbcqa49mib13.cloudfront.net/spark-1.0.1-bin-hadoop2.tgz
sudo tar -zxf spark-1.0.1-bin-hadoop2.tgz
cd spark-1.0.1-bin-hadoop2
# assumes that hostname is set correctly, and the main interface is on eth0, add HOSTNAME to /etc/hosts
sudo su -c "echo `ifconfig eth0 2>/dev/null|awk '/inet addr:/ {print $2}'|sed 's/addr://'` `cat /etc/hostname` >> /etc/hosts"
sbin/start-master.sh
grep -i 'spark://' logs/spark-ubuntu-org*
sudo apt-get update
sudo apt-get install -y openjdk-7-jdk
sudo su -c 'echo "JAVA_HOME=\"/usr/lib/jvm/java-7-openjdk-amd64\"" >> /etc/environment'
cd /opt
sudo wget http://d3kbcqa49mib13.cloudfront.net/spark-1.0.1-bin-hadoop2.tgz
sudo tar -zxf spark-1.0.1-bin-hadoop2.tgz
cd spark-1.0.1-bin-hadoop2
# assumes that hostname is set correctly, and the main interface is on eth0, add HOSTNAME to /etc/hosts
sudo su -c "echo `ifconfig eth0 2>/dev/null|awk '/inet addr:/ {print $2}'|sed 's/addr://'` `cat /etc/hostname` >> /etc/hosts"
echo "spark master url ex: spark://spark-master:1234"
read spark_master_uri
echo "spark master hostname"
read spark_master_hostname
echo "spark master ip"
read spark_master_ip
sudo su -c "echo '$spark_master_ip $spark_master_hostname' >> /etc/hosts"
./bin/spark-class org.apache.spark.deploy.worker.Worker spark_master_uri
./bin/spark-class org.apache.spark.deploy.worker.Worker spark://IP:PORT
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment