dgadiraju

hadoop jar /usr/hdp/2.5.0.0-1245/hadoop-mapreduce/hadoop-mapreduce-examples.jar wordcount \
/public/randomtextwriter \
/user/training/wordcount

dgadiraju

-- Create directory cards under home
mkdir cards
-- Copy largedeck.txt file to the gateway node and place it in cards directory under home directory
-- On lab file is placed under /data directory where any one can read
cp /data/cards/* ~/cards

-- If you want to copy other files you have to use scp/winscp from your PC to gateway

-- Confirm largedeck.txt is available under your home directory
ls -ltr ~/cards

dgadiraju

--Login to mysql CLI
mysql -u retail_dba -h nn01.itversity.com -p

show databases;
use retail_db;
show tables;
select * from departments;

dgadiraju

object hw {
  def main(args: Array[String]) {
    println("Hello World!")
  }
}

dgadiraju

name := "hw"
version := "1.0"
scalaVersion := "2.11.8"

libraryDependencies += "com.typesafe" % "config" % "1.3.1"
libraryDependencies += "mysql" % "mysql-connector-java" % "5.1.36"

dgadiraju

//Immutable
val i = 10 //Smart enough to figure out the data type
val i: Int = 0 //Data type can be defined explicitly as well
//This does not work i = i + 1, as i is defined as immutable (val)

//Mutable
var j = 20
j = j + 1

dgadiraju

//Expression
println("********")
println("Expression")
val c = {
  val i = (math.random * 100).toInt
  val j = (math.random * 100).toInt
  i - j
}
println(c)

dgadiraju

//Creating functions

def addIntegers(i: Int, j: Int): Int = {
  i + j
}

//Anonymous function assigned to a variable
val addIntegers = (i: Int, j: Int) => {
  i + j
}

dgadiraju

// closure example
def m2: Int => Int = {
  val factor = 2
  val multiplier = (i: Int) => i * factor
  multiplier
}

dgadiraju

val inputPath = "/Users/itversity/Research/data/wordcount.txt" or val inputPath = "/public/randomtextwriter/part-m-00000"
val outputPath = "/Users/itversity/Research/data/wordcount" or val outputPath = "/user/dgadiraju/wordcount"
//Make sure outputPath does not exist for this example

sc.textFile(inputPath).
  flatMap(_.split(" ")).
  map((_, 1)).
  reduceByKey(_ + _).
  take(100).
  foreach(println)