Skip to content

Instantly share code, notes, and snippets.

View urlgrey's full-sized avatar

Scott Kidder urlgrey

  • San Francisco, California
View GitHub Profile
diff --git a/bin/server_start.sh b/bin/server_start.sh
index fd4f492..e5b6b9f 100755
--- a/bin/server_start.sh
+++ b/bin/server_start.sh
@@ -26,7 +26,7 @@ GC_OPTS="-XX:+UseConcMarkSweepGC
# To truly enable JMX in AWS and other containerized environments, also need to
# -Djava.rmi.server.hostname equal to the hostname in that environment. This i
# depending on AWS vs GCE etc.
-JAVA_OPTS="-XX:MaxDirectMemorySize=512M \
+JAVA_OPTS="-XX:MaxDirectMemorySize=2048M \
git pull \
&& git checkout tags/v0.5.2
git reset -- bin/server_start.sh \
&& git checkout bin/server_start.sh
<project>
<properties>
<scala.version>2.10.3</scala.version>
<scala.compat.version>2.10</scala.compat.version>
<spark.version>1.3.1</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>org.scala-lang</groupId>
# Template for a Spark Job Server configuration file
# When deployed these settings are loaded when job server starts
#
# Spark Cluster / Job Server configuration
spark {
# spark.master will be passed to each job's JobContext
# master = "local[4]"
master = "spark://spark-master-host:7077"
# master = "mesos://vm28-hulk-pub:5050"
# master = "yarn-client"
curl --data-binary @my-app.jar http://spark-master-host:8090/jars/my-app
~/spark-jobserver/bin/server_start.sh
diff --git a/bin/server_start.sh b/bin/server_start.sh
index 5598ebd..c6e017a 100755
--- a/bin/server_start.sh
+++ b/bin/server_start.sh
@@ -15,13 +15,13 @@ get_abs_script_path
GC_OPTS="-XX:+UseConcMarkSweepGC
-verbose:gc -XX:+PrintGCTimeStamps -Xloggc:$appdir/gc.out
- -XX:MaxPermSize=512m
+ -XX:MaxPermSize=2048m
git clone https://github.com/spark-jobserver/spark-jobserver.git
cd spark-jobserver
git checkout 642bb47cd04906c06d2481a391e8eb2cc30ba4e4
export VER=`sbt version | tail -1 | cut -f2`
sbt assembly
mv /home/ec2-user/spark-jobserver/job-server-extras/target/scala-2.10/spark-job-server.jar ./bin/
cd bin
echo "export SPARK_HOME=/root/spark" > settings.sh
curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo
sudo yum install sbt