Skip to content

Instantly share code, notes, and snippets.

View MDIB's full-sized avatar

Michel Dib MDIB

  • Nubank
  • Berlin, Germany
View GitHub Profile
@MDIB
MDIB / comparision.scala
Last active September 17, 2019 18:19
Spark Schema Comparison
import org.apache.spark.sql.types._
import cats._
import cats.implicits._
sealed trait ComparationResult
case object Validated extends ComparationResult
case class Invalid(msg: String) extends ComparationResult
def safeGetColumn(dfSchema: StructType,columnName: String) =
try{
JSON.stringify($.map($('.expense .summary .involved'),
(e) => {
var negative = $(e).find('.you > .negative').text()
var positive = $(e).find('.you > .positive').text()
var date = $(e).find('.date').prop('title')
var isPositive = (negative == "")
var final = isPositive?positive:negative
var desc = $(e).find('.description').text()
return {amount: final,
desc: desc.trim(),
JSON.stringify(
$.map($('.item_content'),
(e) => {
return {title: $(e).find('.title').text(),
url: $(e).find('.original_url').prop('href'),
image: $(e).find('.lazy-active').css('background-image'),
id: $(e).parent().prop('id')}
})
)