Skip to content

Instantly share code, notes, and snippets.

@ahoy-jon
Created February 26, 2019 10:40
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save ahoy-jon/eef09a9203c66db61f04f2337845dfb2 to your computer and use it in GitHub Desktop.
Save ahoy-jon/eef09a9203c66db61f04f2337845dfb2 to your computer and use it in GitHub Desktop.
val ss: SparkSession
val df: DataFrame
import ss.implicits._
trait NotTypeColumn[C]
object NotTypeColumn {
implicit def notc[C <: org.apache.spark.sql.Column]:NotTypeColumn[C] = new NotTypeColumn[C] {}
implicit def notc1[A,B]:NotTypeColumn[org.apache.spark.sql.TypedColumn[A,B]] = ???
implicit def notc2[A,B]:NotTypeColumn[org.apache.spark.sql.TypedColumn[A,B]] = ???
}
object Ops {
implicit class ColumOps[C <: org.apache.spark.sql.Column]
(val c: C)(implicit not:NotTypeColumn[C]) {
def |>[A, B](f: A => B): org.apache.spark.sql.TypedColumn[Any, B] = ???
}
implicit class TypeColumnOps[A, B](val tc: org.apache.spark.sql.TypedColumn[A, B]) {
def |>[C](f: B => C): org.apache.spark.sql.TypedColumn[A, C] = ???
}
}
import Ops._
df.select($"name".as[String] |> (_ + "!"))
df.select($"name" |> ((x: String) => x + "!"))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment