SparkScalaWordCount.scala

/**
  * Created by malai on 2/1/16.
  */
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._

object WordCount {

  def main(args:Array[String]) : Unit = {

    val logFile = "/home/malai/spark-1.6.0/README.md"
    val conf = new SparkConf().setAppName("Simple Application")
    val sc = new SparkContext(conf)
    val input = sc.textFile(logFile)
    val words = input.flatMap(line => line.split(" "))
    val counts = words.map(word => (word,1)).reduceByKey{case(x,y) => x + y}
    counts.saveAsTextFile("/home/malai/spark-1.6.0/README_log1.md")

  }
}