frosforever

import shapeless._
import shapeless.ops.hlist

object mapper extends Poly1 {
  implicit def caseOpt[T] = at[Option[T]](_ => Option.empty[T])
}

def foo[S, SRep <: HList, O <: HList](s: S)(implicit
  labelledGeneric: Generic.Aux[S, SRep],
  map: hlist.Mapper.Aux[mapper.type, SRep, O],

frosforever

import fs2.{ Pipe, Pull } // Version 1.0.5

import cats.effect.{ ExitCode, IO, IOApp }

object BreakingPipe extends IOApp {

  def onlyOnNonEmpty[F[_], A, B](innerPipe: fs2.Pipe[F, A, B]): fs2.Pipe[F, A, B] = { s =>
    s.pull.peek.flatMap {
      case None          => Pull.pure(None)
      case Some((_, ss)) => ss.through(innerPipe).pull.echo

frosforever

import org.http4s.circe._
import org.http4s.dsl.io._
import org.http4s.implicits._
import org.http4s.server.blaze.BlazeServerBuilder
import org.http4s.{ HttpApp, HttpRoutes, Request }

import cats.data.Kleisli
import cats.effect._
import cats.effect.concurrent.Ref
import cats.implicits._

frosforever

// See https://docs.aws.amazon.com/athena/latest/ug/application-load-balancer-logs.html for regex pattern and column names

val raw = spark.read.text("s3://some_s3_path/albname/AWSLogs/accountId/elasticloadbalancing/region/year/month/day/")

val regex = """([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*)[:-]([0-9]*) ([-.0-9]*) ([-.0-9]*) ([-.0-9]*) (|[-0-9]*) (-|[-0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) (.*) (- |[^ ]*)\" \"([^\"]*)\" ([A-Z0-9-_]+) ([A-Za-z0-9.-]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^\"]*)\" ([-.0-9]*) ([^ ]*) \"([^\"]*)\" \"([^\"]*)\" \"([^ ]*)\" \"([^\s]+?)\" \"([^\s]+)\" \"([^ ]*)\" \"([^ ]*)\""""

val albLogs = raw.select(
   regexp_extract($"value", regex, 1).as("type"),
    regexp_extract($"value", regex, 2).as("time"),
    regexp_extract($"value", regex, 3).as("elb"),