Skip to content

Instantly share code, notes, and snippets.

@HeartSaVioR
Created July 14, 2015 08:32
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save HeartSaVioR/eb60eca01782c2c2b0c6 to your computer and use it in GitHub Desktop.
Save HeartSaVioR/eb60eca01782c2c2b0c6 to your computer and use it in GitHub Desktop.
build.sbt for Spark 1.4.0 + HBase CDH 5.2.0 + elasticsearch-spark 2.1.0 + and so on
import AssemblyKeys._
name := "elasticsearch-spark-project"
version := "1.0-SNAPSHOT"
scalaVersion := "2.10.5"
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % "1.4.0" % "provided",
"org.apache.hadoop" % "hadoop-client" % "2.5.0-cdh5.2.0" % "provided",
"org.apache.hbase" % "hbase-common" % "0.98.6-cdh5.2.0"
exclude("tomcat", "jasper-compiler")
exclude("tomcat", "jasper-runtime")
exclude("org.apache.hadoop", "hadoop-annotation"),
"org.apache.hbase" % "hbase-client" % "0.98.6-cdh5.2.0"
exclude("org.apache.hadoop", "hadoop-annotation")
exclude("tomcat", "jasper-compiler")
exclude("tomcat", "jasper-runtime")
exclude("io.netty", "netty"),
"org.apache.hbase" % "hbase-server" % "0.98.6-cdh5.2.0"
exclude("org.apache.hadoop", "hadoop-core")
exclude("org.apache.hadoop", "hadoop-annotation")
exclude("org.apache.hadoop", "hadoop-client")
exclude("org.apache.hadoop", "hadoop-mapreduce-client-jobclient")
exclude("org.apache.hadoop", "hadoop-mapreduce-client-core")
exclude("org.apache.hadoop", "hadoop-auth")
exclude("org.apache.hadoop", "hadoop-annotations")
exclude("org.apache.hadoop", "hadoop-hdfs")
exclude("org.apache.hbase", "hadoop-hadoop1-compat")
exclude("org.apache.commons", "commons-math")
exclude("com.sun.jersey", "jersey-core")
exclude("com.sun.jersey", "jersey-server")
exclude("com.sun.jersey", "jersey-json")
exclude("org.slf4j", "slf4j-api")
exclude("commons-io", "commons-io")
exclude("tomcat", "jasper-compiler")
exclude("tomcat", "jasper-runtime"),
"org.json4s" %% "json4s-jackson" % "3.2.11",
"com.github.nscala-time" %% "nscala-time" % "2.0.0",
// persistance
"com.typesafe.slick" %% "slick" % "2.1.0",
"mysql" % "mysql-connector-java" % "5.1.24",
"com.mchange" % "c3p0" % "0.9.5",
// elasticsearch-spark, it only supports scala version 2.10 officially
// https://www.elastic.co/guide/en/elasticsearch/hadoop/current/install.html#_minimalistic_binaries
"org.elasticsearch" %% "elasticsearch-spark" % "2.1.0" % "provided",
"org.scalatest" %% "scalatest" % "2.2.4" % "test"
)
dependencyOverrides += "com.google.guava" % "guava" % "12.0"
dependencyOverrides += "commons-codec" % "commons-codec" % "1.10"
// If using CDH, also add Cloudera repo
resolvers ++= Seq(
"Cloudera repos" at "https://repository.cloudera.com/artifactory/cloudera-repos"
)
resolvers += Resolver.sonatypeRepo("public")
resolvers += Resolver.mavenLocal
mergeStrategy in assembly := { (old) => old match {
case PathList("javax", "servlet", xs@_*) => MergeStrategy.first
case PathList("javax", "transaction", xs@_*) => MergeStrategy.first
case PathList("javax", "mail", xs@_*) => MergeStrategy.first
case PathList("javax", "activation", xs@_*) => MergeStrategy.first
case PathList(ps@_*) if ps.last endsWith ".html" => MergeStrategy.first
case "application.conf" => MergeStrategy.concat
case "unwanted.txt" => MergeStrategy.discard
case x => defaultMergeStrategy(x)
}
}
net.virtualvoid.sbt.graph.Plugin.graphSettings
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment