avoid sbt assemblies

Build.scala

 ...
 
 // define your projects, with a custom "settings"
  lazy val common = Project(
    id = "common",
    base = file("common"),
    settings = baseSettings ++ Seq(
      libraryDependencies ++= commonLibraryDependencies
    )
  )
  
  ...
  
  //some git helpers in sbt.  we only care about gitSha for this particular demo, but they might all be useful
  
  // make sure we get multiline and newline cleaned up
  def trimLine(line: String) = {
    val l = line.replaceAll("\n", " ").trim; l.substring(0, Math.min(l.length, 45))
  }

  lazy val gitSha = trimLine("git rev-parse --short HEAD" !!)
  lazy val gitDiffTmp = trimLine("git diff --shortstat" !!)
  lazy val gitStatus = gitDiffTmp + (if (gitDiffTmp.size == 0) {
    "clean"
  } else {
    ""
  })
  lazy val gitDescr = trimLine("git describe --tags --always" !!) + (if (gitDiffTmp.size > 0) {
    "-dirty"
  } else {
    ""
  })
  //emulating --dirty as it is only in git >1.7
  lazy val gitShaClean = gitSha + (if (gitDiffTmp.size == 0) {
    ""
  } else {
    "-dirty"
  })



 
 //create a task
 lazy val exportClasspath = taskKey[Unit]("Dump classpath to files")
 
 // define what the task actually does inside the "settings"
 //
 // the important part of this task is that it creates a file
 //  <project>/target/managed_dependency_sha-<git-sha>.txt
 // which takes all the managed dependencies and computes the sha of their name
 lazy val baseSettings = Defaults.defaultSettings ++ assemblySettings ++ Revolver.settings ++ Seq(
    exportClasspath := {
      val managedDir: File = (managedDirectory in Runtime value)
      val mP = managedDir.getAbsolutePath
      val unmanagedDir: File = (unmanagedBase in Runtime value)
      val uP = unmanagedDir.getAbsolutePath
      val cp: Seq[File] = (fullClasspath in Runtime value).files
      val projectName: String = (name in Runtime value)

      @annotation.tailrec
      def filterByDir(files: Seq[String], dirs: Seq[String], acc: Map[String,Seq[String]]):
      (Map[String, Seq[String]], Seq[String]) = {
        if (dirs.length >= 1) {
          val d = dirs.head
          val (good, bad) = files.partition{_.startsWith(d)}
          filterByDir(bad, dirs.tail, acc + (dirs.head -> good.map{_.substring(d.length)}))
        } else {
          (acc,files)
        }
      }

      val (matched, ignored) = filterByDir(cp.map{_.getAbsolutePath}, Seq(mP, uP), Map())
      println(ignored.size + " classpath entries were not in expected dir, ignoring:")
      ignored.foreach{i => println("\t" + i)}

      def sha(s: String): String = {
        val bytes = java.security.MessageDigest.getInstance("SHA").digest(s.getBytes)
        bytes.map("%02X".format(_)).mkString
      }

     def sortShaAndExport(files: Seq[String], out:String): String = {
        //Note: you might *not* want to sort the files, as the order can affect conflict resolution
        // this only makes sense if you really know order doesn't matter
        val s = files.sorted
        val theSha = sha(s.mkString(","))
        val o = new java.io.PrintWriter(projectName + "/target/" + out + "_classpath.txt")
        s.foreach{o.println(_)}
        o.close()
        val o2 = new java.io.PrintWriter(projectName + "/target/" + out + "_dependency_sha-" +
          gitSha + ".txt")
        o2.println(theSha)
        o2.close()
        theSha
      }

      sortShaAndExport(matched(mP), "managed")
      sortShaAndExport(matched(uP), "unmanaged")
    }
  )

build.sh

#!/bin/bash

SCRIPT_DIR=`dirname $0`
source "$SCRIPT_DIR/utils.sh"

while [ "$1" != "" ]; do
    case $1 in
    "-project")
        shift
        MAIN_PROJECT=$1
    ;;
    "-host")
        shift
        HOST=$1
    ;;
    "-user")
        shift
        REMOTE_USER=$1
    ;;  
    "-targetDir")
        shift
        TARGET_DIR=$1
    ;;  
    "-allowDirty")
        # only use for debugging
        ALLOW_DIRTY=1
    ;;
    esac
    shift
done

SHA=`git rev-parse --short HEAD`
BRANCH=`git rev-parse --abbrev-ref HEAD`
STATUS=`git status --porcelain`

if [[ -n "$STATUS" && -z $ALLOW_DIRTY ]]; then
  die "git status is not clean"
fi

if [ -z $MAIN_PROJECT ]; then
  MAIN_PROJECT=transform
fi

if [ -z $REMOTE_USER ]; then
  REMOTE_USER=dev
fi

if [ -z $HOST ]; then
  if [ $REMOTE_USER = "dev" ]; then
    HOST="driver.dev.quantifind.com"
  elif [ $REMOTE_USER = "prod" ]; then
    HOST="driver.prod.quantifind.com"
  else
    die "can't guess host from user $REMOTE_USER"
  fi
fi

if [ -z $TARGET_DIR ]; then
  TARGET_DIR="/home/$REMOTE_USER/jars"
fi


LOCAL_TARGET_DIR="output/$BRANCH"
REMOTE_DEST_DIR="$TARGET_DIR/$BRANCH"
REMOTE="$REMOTE_USER@$HOST"


echo "Invoking SBT to determine project dependency graph"
DEP_ARRAY=($(sbt/sbt "project all" printProjectToDeps 2>&1 >/dev/null))

echo "Computing transitive dependences for $MAIN_PROJECT"
function transitive() {
  PROJECTS=($1)
  TODO=($1)
  while [ ${#TODO[@]} != 0 ]; do
    CURRENT=${TODO[0]}
    TODO=("${TODO[@]:1}")
    for entry in "${DEP_ARRAY[@]}"; do
      if [ "$CURRENT" == "${entry%%:*}" ]; then 
        VALUES=${entry#*:}
        OIFS=${IFS}
        IFS=","
        for VALUE in $VALUES; do
          TODO+=($VALUE)
          PROJECTS+=($VALUE)
        done
        IFS=${OIFS}
      fi
    done # added dependencies for CURRENT
  done # TODO is empty
}
transitive $MAIN_PROJECT

# removing duplicates
PROJECTS=($(tr ' ' '\n' <<< "${PROJECTS[@]}" | sort -u | tr '\n' ' '))
echo "Main project $MAIN_PROJECT depends on:"
printf -- '%s\n' "${PROJECTS[@]}"
echo

mkdir -p "output/$BRANCH"
echo "packaging ${#PROJECTS[@]} projects (${PROJECTS[@]}) to $REMOTE:$REMOTE_DEST_DIR"
echo "bundling deps of $MAIN_PROJECT to $REMOTE:$REMOTE_DEST_DIR"
date

# TODO: maybe the whole thing should be in sbt

REBUILD=()
for PROJECT in "${PROJECTS[@]}"
do
  # because we only build from clean state, if the jar already exists we don't need to rebuild
  LOCAL_JAR_PATH="$LOCAL_TARGET_DIR/$PROJECT-package-$SHA.jar"
  if [ ! -f $LOCAL_JAR_PATH ]; then
    REBUILD+=($PROJECT)        
  fi
done

SHA_FILE="${MAIN_PROJECT}/target/managed_dependency_sha-${SHA}.txt"
if [ ${#REBUILD[@]} -eq 0 ]; then
  echo "No packages need to be rebuilt"
else 
  BUILD_CMD="sbt/sbt"
  for PROJECT in "${REBUILD[@]}"
  do
    BUILD_CMD="$BUILD_CMD \"project $PROJECT\" package"
  done
  # also export the sha of the deps, to check if it needs to be updated
  if [ -f $SHA_FILE ]; then
    echo "sha file $SHA_FILE already exists"
  else
    echo "computing dependency sha to $SHA_FILE"
    BUILD_CMD="$BUILD_CMD \"project $MAIN_PROJECT\" exportClasspath"
  fi 
  echo $BUILD_CMD
  eval $BUILD_CMD || die "error while packaging projects ${REBUILD[@]}"
fi

for PROJECT in "${PROJECTS[@]}"
do
  LOCAL_JAR_PATH="$LOCAL_TARGET_DIR/$PROJECT-package-$SHA.jar"
  echo "moving package for $PROJECT to destination"
  if [ -f $LOCAL_JAR_PATH ]; then
    echo "$LOCAL_JAR_PATH already exists"
  else
    cp "$PROJECT/target/scala-2.10/${PROJECT}_2.10-0.1-SNAPSHOT.jar" $LOCAL_JAR_PATH
  fi
  copyToRemote $LOCAL_JAR_PATH $REMOTE $REMOTE_DEST_DIR "$PROJECT-package-$SHA.jar"
done

DEPENDENCY_SHA=`cat $SHA_FILE`
echo "dep sha is $DEPENDENCY_SHA"

DEP_FILE="output/${MAIN_PROJECT}-deps-${DEPENDENCY_SHA}.jar"
if [ -f $DEP_FILE ]; then
  echo "dependency file $DEP_FILE already exists locally"
else
  echo "building dependency file"
  sbt/sbt "project $MAIN_PROJECT" assembly-package-dependency || die "error while packaging dependencies"
  cp "$MAIN_PROJECT/target/scala-2.10/$MAIN_PROJECT-assembly-0.1-SNAPSHOT-deps.jar" $DEP_FILE
fi

# Though locally we don't put the deps into a branched dir, on the remote we will, just
# because we often used a shared dir, and since we usually just grab the "latest" file,
#  somebody else could deploy some deps and totally screw you up w/out you noticing
copyToRemote $DEP_FILE $REMOTE $REMOTE_DEST_DIR "${MAIN_PROJECT}-deps-${DEPENDENCY_SHA}.jar"

utils.sh

#!/bin/bash

die() { echo "$@" 1>&2 ; exit 1; }

## in hindsight, this should just use rsync
copyToRemote() {
  LOCAL_FILE=$1
  REMOTE=$2
  REMOTE_DIR=$3
  REMOTE_FILE=$4
  REMOTE_FULL_PATH="$3/$4"
  if ssh -q "$REMOTE" [[ -f $REMOTE_FULL_PATH ]]; then
    echo "$REMOTE:$REMOTE_FULL_PATH already exists"
  else
    echo "deploying to $REMOTE:$REMOTE_FULL_PATH"
    ssh "$REMOTE" "mkdir -p $REMOTE_DIR" || die "couldn't make remote dirs $REMOTE_DIR"
    scp $LOCAL_FILE "$REMOTE:$REMOTE_FULL_PATH" || die "couldn't scp jar"
  fi
}