urlgrey

### Keybase proof

I hereby claim:

  * I am urlgrey on github.
  * I am urlgrey (https://keybase.io/urlgrey) on keybase.
  * I have a public key whose fingerprint is 3448 2319 82F6 AF7A 0A9F  3ECC 40E8 5AC5 269B 8818

To claim this, I am signing this object:

urlgrey

mkdir -p ~/lib && \
cd ~/lib && \
wget http://central.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/3.4.1/stanford-corenlp-3.4.1.jar && \
wget http://central.maven.org/maven2/edu/stanford/nlp/stanford-corenlp/3.4.1/stanford-corenlp-3.4.1-models.jar

urlgrey

sudo bash -c 'echo "spark.executor.extraClassPath /root/ephemeral-hdfs/conf:/home/ec2-user/lib/stanford-corenlp-3.4.1.jar:/home/ec2-user/lib/stanford-corenlp-3.4.1-models.jar" >> /root/spark/conf/spark-defaults.conf'

urlgrey

curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo
sudo yum install sbt

urlgrey

git clone https://github.com/spark-jobserver/spark-jobserver.git
cd spark-jobserver
git checkout 642bb47cd04906c06d2481a391e8eb2cc30ba4e4
export VER=`sbt version | tail -1 | cut -f2`
sbt assembly
mv /home/ec2-user/spark-jobserver/job-server-extras/target/scala-2.10/spark-job-server.jar ./bin/
cd bin
echo "export SPARK_HOME=/root/spark" > settings.sh

urlgrey

diff --git a/bin/server_start.sh b/bin/server_start.sh
index 5598ebd..c6e017a 100755
--- a/bin/server_start.sh
+++ b/bin/server_start.sh
@@ -15,13 +15,13 @@ get_abs_script_path

 GC_OPTS="-XX:+UseConcMarkSweepGC
          -verbose:gc -XX:+PrintGCTimeStamps -Xloggc:$appdir/gc.out
-         -XX:MaxPermSize=512m
+         -XX:MaxPermSize=2048m

urlgrey

~/spark-jobserver/bin/server_start.sh

urlgrey

curl --data-binary @my-app.jar http://spark-master-host:8090/jars/my-app

urlgrey

# Template for a Spark Job Server configuration file
# When deployed these settings are loaded when job server starts
#
# Spark Cluster / Job Server configuration
spark {
  # spark.master will be passed to each job's JobContext
  # master = "local[4]"
  master = "spark://spark-master-host:7077"
  # master = "mesos://vm28-hulk-pub:5050"
  # master = "yarn-client"

urlgrey

<project>
  <properties>
	<scala.version>2.10.3</scala.version>
	<scala.compat.version>2.10</scala.compat.version>
	<spark.version>1.3.1</spark.version>
  </properties>

	<dependencies>
		<dependency>
			<groupId>org.scala-lang</groupId>