Gradle buildscript for Spark shell scripts - Unibo BigData course 2019

build.gradle

plugins {
    // Apply the scala plugin to add support for Scala
    id 'scala'
}

group 'it.unibo'
version '1.0-SNAPSHOT'

def javaVersion = JavaVersion.VERSION_1_7
def scalaBinaryVersion = "2.11"
def scalaMinorVersion = "12"
def scalaVersion = "$scalaBinaryVersion.$scalaMinorVersion"

def clouderaVersion = "cdh5.13.1"
def hadoopVersion = "2.6.0"
def sparkVersion = "2.1.0"

repositories {
    // Local Maven repo
    mavenLocal()

    // Standard Maven central repos
    jcenter()
    mavenCentral()

    // Cloudera Release Repo needed for those specific hadoop libraries versions
    // (probably not needed if using other versions)
    maven { url "https://repository.cloudera.com/content/repositories/releases/" }

    // Cloudera Third-Party Repo needed for Cloudera-specific dependencies
    // (probably not needed if using other versions)
    maven { url "https://repository.cloudera.com/content/repositories/third-party/" }
}

dependencies {
    implementation "org.scala-lang:scala-library:$scalaVersion"
    implementation "org.scala-lang:scala-reflect:$scalaVersion"
    implementation "org.scala-lang:scala-compiler:$scalaVersion"

    // Use Scalatest for testing our library
    testImplementation "junit:junit:4.12"
    testImplementation "org.scalatest:scalatest_$scalaBinaryVersion:3.0.6"

    // Need scala-xml at test runtime
    testRuntimeOnly "org.scala-lang.modules:scala-xml_$scalaBinaryVersion:1.1.1"

    // https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common
    implementation "org.apache.hadoop:hadoop-common:$hadoopVersion-$clouderaVersion"

    // https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-core
    implementation "org.apache.hadoop:hadoop-core:$hadoopVersion-mr1-$clouderaVersion"

    // https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs
    implementation "org.apache.hadoop:hadoop-hdfs:$hadoopVersion-$clouderaVersion"

    // Dependencies from https://github.com/faizanahemad/spark-gradle-template/blob/master/build.gradle
    implementation "org.apache.spark:spark-mllib_$scalaBinaryVersion:$sparkVersion"
    implementation "org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkVersion"
    implementation "org.apache.spark:spark-streaming-flume-assembly_$scalaBinaryVersion:$sparkVersion"
    implementation "org.apache.spark:spark-graphx_$scalaBinaryVersion:$sparkVersion"
    implementation "org.apache.spark:spark-launcher_$scalaBinaryVersion:$sparkVersion"
    implementation "org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkVersion"
    implementation "org.apache.spark:spark-streaming_$scalaBinaryVersion:$sparkVersion"
    implementation "org.apache.spark:spark-core_$scalaBinaryVersion:$sparkVersion"

    implementation "commons-io:commons-io:2.5"
}

java {
    sourceCompatibility = javaVersion
    targetCompatibility = javaVersion
}