tonyfraser

// run from spark shell
// you'd probably never do this, but just in case you ever wanted to.

import scala.io.Source
import spark.implicits

val url = "https://raw.githubusercontent.com/sitepoint-editors/json-examples/master/src/db.json"

val json = Source.
  fromURL(url).

tonyfraser

import spark.implicits

val df = Seq(
  ("bravo", "southern charm", "b-sc-first-episode", true, false, 5, 10),
  ("bravo", "southern charm", "b-sc-second-episode", false, false, 11, 22),
  ("bravo", "vanderpump", "b-v-first-episode", true, false, 3, 6),
  ("bravo", "vanderpump", "b-v-second-episode", false,false, 4, 8),
  ("syfy", "krypton", "s-kr-first-episode", false, true, 2, 4),
  ("syfy", "below deck", "s-bd-first-episode", true, true, 1, 2)
).toDF("network_name", "show_name", "episode", "in_scope", "supported", "completes", "views").

tonyfraser

// -------------------------------------------------------------
// spark/scala
// -------------------------------------------------------------

//union a dataframe and return the records that a are different
val diff = df.union(df2).except(df))

// -------------------------------------------------------------

//udf for turning empty dataframe cells into null dataframe cells

tonyfraser

import org.apache.spark.sql.functions._
import org.apache.spark.sql.Row
import spark.implicits
import org.apache.spark.sql.functions.udf

// https://stackoverflow.com/questions/55083109/how-to-get-min-and-max-value-from-multiple-columns-in-a-dataframe-in-spark

// assumes the first column is an index and probably a string, but all other columns are integers.
val df = Seq(
    ("r0", 0, 2, 3),

tonyfraser

import java.time.format.DateTimeFormatter
import java.time.LocalDateTime
import java.time.ZoneId

val ny = ZoneId.of("America/New_York")
val utc = ZoneId.of("UTC")
val dateTime = LocalDateTime.now.atZone(utc)

val nyTime = DateTimeFormatter.
    ofPattern("yyyy-MMM-dd HH:mm z").

tonyfraser

package object mail {

  implicit def stringToSeq(single: String): Seq[String] = Seq(single)
  implicit def liftToOption[T](t: T): Option[T] = Some(t)

  sealed abstract class MailType
  case object Plain extends MailType
  case object Rich extends MailType
  case object MultiPart extends MailType