Scala Spark with sbt-assembly example configuration
#!/usr/bin/env bash | |
set -o nounset | |
set -o errexit | |
readonly default_env="${0#*-}" | |
readonly ENV=${1:-$default_env} | |
readonly JAR_NAME="your-analytics.jar" | |
readonly UPLOAD_JAR=`dirname $0`/../target/scala-2.10/$JAR_NAME | |
case $ENV in | |
integration ) | |
HOST=your-integration-host.compute.amazonaws.com | |
;; | |
prod ) | |
HOST=your-prod-host.compute.amazonaws.com | |
;; | |
* ) | |
echo "environment $ENV not supported" | |
exit 1 | |
;; | |
esac | |
cd `dirname $0`/.. | |
./sbt assembly | |
readonly remote_md5sum=`ssh root@$HOST "/usr/bin/md5sum /root/$JAR_NAME | cut -d ' ' -f1"` | |
readonly local_md5sum=`md5 -q $UPLOAD_JAR` | |
if [ "$remote_md5sum" != "$local_md5sum" ]; then | |
scp $UPLOAD_JAR root@$HOST:~ | |
else | |
echo "MD5 sum $remote_md5sum matched local JAR; not uploading" | |
fi | |
ssh -t root@$HOST "export SPARK_JAVA_OPTS='-Dconfig.resource=$ENV.conf' && /root/spark/bin/spark-shell --jars /root/$JAR_NAME" |
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.12.0") |
sbt.version=0.13.7 |
import sbt._ | |
import sbt.Keys._ | |
import sbtassembly.AssemblyPlugin.autoImport._ | |
object Build extends Build { | |
lazy val analytics = Project( | |
id = "analytics", | |
base = file("."), | |
settings = Seq( | |
name := "analytics", | |
organization := "com.your-name", | |
version := "0.1-SNAPSHOT", | |
scalaVersion := "2.10.3", | |
ivyScala := ivyScala.value map { _.copy(overrideScalaVersion = true) }, | |
assemblyJarName in assembly := "your-analytics.jar", | |
assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false), | |
libraryDependencies ++= Seq( | |
"org.apache.spark" %% "spark-core" % "1.2.0" % "provided", | |
"org.postgresql" % "postgresql" % "9.3-1102-jdbc41", | |
"com.datastax.spark" %% "spark-cassandra-connector" % "1.2.0-alpha1", | |
"com.typesafe" % "config" % "1.2.1", | |
"net.ceedubs" %% "ficus" % "1.0.1", | |
"com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", | |
"com.chuusai" % "shapeless_2.10.3" % "2.0.0" | |
) | |
) | |
) | |
} |
# the binary sbt-launch.jar for 0.13.7 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment