Skip to content

Instantly share code, notes, and snippets.

@potix2
Last active August 29, 2015 14:23
Show Gist options
  • Save potix2/df08e9d8e1bc5cd0e5d0 to your computer and use it in GitHub Desktop.
Save potix2/df08e9d8e1bc5cd0e5d0 to your computer and use it in GitHub Desktop.
class ConcatDFSpec extends org.specs2.mutable.Specification {
def concat(df: DataFrame, inputColName: String, concatColName: String, outputColName: String, dataType: DataType): DataFrame =
df.withColumn(outputColName, callUDF(_ ++ _, dataType, df(inputColName), df(concatColName)))
// val paramMap: ParamMap
// val inputCol: Param[String]
// val outputCol: Param[String]
// val concatCol: Param[String]
// concat(df, paramMap(inputCol), paramMap(outputCol), paramMap(concatCol), dataType)
type A = Map[String, Int]
forAll { (m: List[(A, A)]) => {
val df = TestDataFrame("in1" -> m.map(_._1), "in2" -> m.map(_._2))
val actual = concat(df, "in1", "in2", "out", dataType)
actual must haveSameElementsWith("out")(m.map(_._1 ++ _._2))
}
).check
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment