Scala Spark with sbt-assembly example configuration
|
#!/usr/bin/env bash |
|
|
|
set -o nounset |
|
set -o errexit |
|
|
|
readonly default_env="${0#*-}" |
|
readonly ENV=${1:-$default_env} |
|
readonly JAR_NAME="your-analytics.jar" |
|
readonly UPLOAD_JAR=`dirname $0`/../target/scala-2.10/$JAR_NAME |
|
|
|
case $ENV in |
|
integration ) |
|
HOST=your-integration-host.compute.amazonaws.com |
|
;; |
|
prod ) |
|
HOST=your-prod-host.compute.amazonaws.com |
|
;; |
|
* ) |
|
echo "environment $ENV not supported" |
|
exit 1 |
|
;; |
|
esac |
|
|
|
cd `dirname $0`/.. |
|
./sbt assembly |
|
|
|
readonly remote_md5sum=`ssh root@$HOST "/usr/bin/md5sum /root/$JAR_NAME | cut -d ' ' -f1"` |
|
readonly local_md5sum=`md5 -q $UPLOAD_JAR` |
|
|
|
if [ "$remote_md5sum" != "$local_md5sum" ]; then |
|
scp $UPLOAD_JAR root@$HOST:~ |
|
else |
|
echo "MD5 sum $remote_md5sum matched local JAR; not uploading" |
|
fi |
|
|
|
ssh -t root@$HOST "export SPARK_JAVA_OPTS='-Dconfig.resource=$ENV.conf' && /root/spark/bin/spark-shell --jars /root/$JAR_NAME" |
|
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.12.0") |
|
import sbt._ |
|
import sbt.Keys._ |
|
import sbtassembly.AssemblyPlugin.autoImport._ |
|
|
|
object Build extends Build { |
|
|
|
lazy val analytics = Project( |
|
id = "analytics", |
|
base = file("."), |
|
settings = Seq( |
|
name := "analytics", |
|
organization := "com.your-name", |
|
version := "0.1-SNAPSHOT", |
|
scalaVersion := "2.10.3", |
|
ivyScala := ivyScala.value map { _.copy(overrideScalaVersion = true) }, |
|
assemblyJarName in assembly := "your-analytics.jar", |
|
assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false), |
|
libraryDependencies ++= Seq( |
|
"org.apache.spark" %% "spark-core" % "1.2.0" % "provided", |
|
"org.postgresql" % "postgresql" % "9.3-1102-jdbc41", |
|
"com.datastax.spark" %% "spark-cassandra-connector" % "1.2.0-alpha1", |
|
"com.typesafe" % "config" % "1.2.1", |
|
"net.ceedubs" %% "ficus" % "1.0.1", |
|
"com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", |
|
"com.chuusai" % "shapeless_2.10.3" % "2.0.0" |
|
) |
|
) |
|
) |
|
} |
|
#!/bin/sh |
|
java \ |
|
-Xms512M \ |
|
-Xmx8G \ |
|
-Xss1M \ |
|
-XX:+CMSClassUnloadingEnabled \ |
|
-XX:PermSize=512M \ |
|
-XX:MaxPermSize=2G \ |
|
-jar `dirname $0`/sbt-launch.jar \ |
|
"$@" |
|
# the binary sbt-launch.jar for 0.13.7 |