Create a gist now

Instantly share code, notes, and snippets.

@cfeduke /bin_deploy
Last active Aug 29, 2015

What would you like to do?
Scala Spark with sbt-assembly example configuration
#!/usr/bin/env bash
set -o nounset
set -o errexit
readonly default_env="${0#*-}"
readonly ENV=${1:-$default_env}
readonly JAR_NAME="your-analytics.jar"
readonly UPLOAD_JAR=`dirname $0`/../target/scala-2.10/$JAR_NAME
case $ENV in
integration )
HOST=your-integration-host.compute.amazonaws.com
;;
prod )
HOST=your-prod-host.compute.amazonaws.com
;;
* )
echo "environment $ENV not supported"
exit 1
;;
esac
cd `dirname $0`/..
./sbt assembly
readonly remote_md5sum=`ssh root@$HOST "/usr/bin/md5sum /root/$JAR_NAME | cut -d ' ' -f1"`
readonly local_md5sum=`md5 -q $UPLOAD_JAR`
if [ "$remote_md5sum" != "$local_md5sum" ]; then
scp $UPLOAD_JAR root@$HOST:~
else
echo "MD5 sum $remote_md5sum matched local JAR; not uploading"
fi
ssh -t root@$HOST "export SPARK_JAVA_OPTS='-Dconfig.resource=$ENV.conf' && /root/spark/bin/spark-shell --jars /root/$JAR_NAME"
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.12.0")
sbt.version=0.13.7
import sbt._
import sbt.Keys._
import sbtassembly.AssemblyPlugin.autoImport._
object Build extends Build {
lazy val analytics = Project(
id = "analytics",
base = file("."),
settings = Seq(
name := "analytics",
organization := "com.your-name",
version := "0.1-SNAPSHOT",
scalaVersion := "2.10.3",
ivyScala := ivyScala.value map { _.copy(overrideScalaVersion = true) },
assemblyJarName in assembly := "your-analytics.jar",
assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false),
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % "1.2.0" % "provided",
"org.postgresql" % "postgresql" % "9.3-1102-jdbc41",
"com.datastax.spark" %% "spark-cassandra-connector" % "1.2.0-alpha1",
"com.typesafe" % "config" % "1.2.1",
"net.ceedubs" %% "ficus" % "1.0.1",
"com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2",
"com.chuusai" % "shapeless_2.10.3" % "2.0.0"
)
)
)
}
#!/bin/sh
java \
-Xms512M \
-Xmx8G \
-Xss1M \
-XX:+CMSClassUnloadingEnabled \
-XX:PermSize=512M \
-XX:MaxPermSize=2G \
-jar `dirname $0`/sbt-launch.jar \
"$@"
# the binary sbt-launch.jar for 0.13.7
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment