Spark-asignments

Problem1.text

val rdd_1 = sc.parallelize(Seq((1, 3.6)))
  val rdd_2 = sc.parallelize(Seq((1, 1.1)))

  println(s"Wanted Result:", findSubOfVAlues(rdd_1, rdd_2))

  def findSubOfVAlues(value: RDD[(Int, Double)], value1: RDD[(Int, Double)]) = {
    val combordd = value join value1

    val result = combordd.map(r => (r._1, r._2._1 - r._1, r._2._2)).collect

    println(result)
  }
  
  
  Output:
  
  Array[(Int, Double)] = Array((1,2.5))

Problem2.text

  val rdd1 = sc.parallelize(Array((1, Array((3, 4), (4, 5))), (2, Array((4, 2), (4, 4), (3, 9)))))

  val result = rdd1.flatMapValues(value => value).collect
  
  
   Output:
   
   (1,(3,4)), (1,(4,5)), (2,(4,2)), (2,(4,4)), (2,(3,9))

Problem3.text

val rdd2 = sc.parallelize(Array((1, List(1, 2, 3, 4)), (2, List(1, 2, 3, 4)), (3, List(1, 2, 3, 4)), (4, List(1, 2, 3, 4))))

val result = rdd2.flatMapValues(value => value).filter(x => x._1 == x._2)

Output:


(1,1), (2,2), (3,3), (4,4)

sparks Assignment for Date: 10-08-2018