Implement spark.rdd.pipe

pipe.scala

import scala.io.Source
import java.io.PrintWriter

val command = "/tmp/cmd.sh"
val proc = Runtime.getRuntime.exec(Array(command))
new Thread("stderr reader for " + command) {
  override def run() {
    for(line <- Source.fromInputStream(proc.getErrorStream).getLines)
      System.err.println(line)
  }
}.start()

val lineList = List("hello","how","are","you")
new Thread("stdin writer for " + command) {
  override def run() {
    val out = new PrintWriter(proc.getOutputStream)
    for(elem <- lineList)
      out.println(elem)
    out.close()
  }
}.start()

val outputLines = Source.fromInputStream(proc.getInputStream).getLines
println("======> " + outputLines.toList)