sderosiaux

spark-submit --master yarn --deploy-mode cluster # yarn cluster mode (driver in yarn)
    --conf spark.yarn.maxAppAttempts=4 # default 2
    --conf spark.yarn.am.attemptFailuresValidityInterval=1h # reset the count every hour (a streaming app can last months)
    --conf spark.yarn.max.executor.failures={8 * num_executors} # default is max(2*num_executors, 3). So for 4 executors: 32 against 8.
    --conf spark.yarn.executor.failuresValidityInterval=1h # same as before, but for the executors
    --conf spark.task.maxFailures=8 # default is 4
    --queue realtime_queue # do not mess with the default queue
    --conf spark.speculation=true # ensure the job is idempotent (it will start the same job twice if the first is slow)
    --conf spark.driver.extraJavaOptions=-Dlog4j.configuration=file:log4j.properties # eh, logging to some logstash for instance
    --conf spark.executor.extraJavaOptions=-Dlog4j.configuration=file:log4j.properties

sderosiaux

let types = /private/tmp/dhall-kubernetes/types.dhall
let defaults = /private/tmp/dhall-kubernetes/defaults.dhall

let deployment: types.Deployment =
    defaults.Deployment
    //{
        metadata = defaults.ObjectMeta //{ name = "nginx" },
        spec = Some (
            defaults.DeploymentSpec
            //{

sderosiaux

object Tree {
  sealed trait Tree[A]
  case class Leaf[A](a: A) extends Tree[A]
  case class Node[A](left: Tree[A], right: Tree[A]) extends Tree[A]
  
  def translate[A, B, C, F[_], E](t: Tree[A], check: (A, B) => Boolean, newState: B => B, endState: (A, B) => C, err: A => E)(implicit F: MonadError[F, E]): StateT[F, B, Tree[C]] = t match {
    case Leaf(a) => StateT(s => if (check(a, s)) F.raiseError(err(a)) else F.pure((newState(s), Leaf(endState(a, s)))))
    case Node(left, right) =>
      implicit val A = Apply[StateT[F, B, ?]]
      (translate(left, check, newState, endState, err), translate(right, check, newState, endState, err)).mapN(Node.apply)

sderosiaux

Keybase proof

I hereby claim:

• I am sderosiaux on github.
• I am chtefi (https://keybase.io/chtefi) on keybase.
• I have a public key whose fingerprint is F6D8 1D9C D7E8 0CD7 1DB0 34B0 51D3 1B66 50EB 283A

To claim this, I am signing this object:

sderosiaux

def forLoop[A, C](
                   from: A,
                   continue: A => Boolean,
                   next: A => A,
                   compute: (C, A) => C,
                   acc: C): C = {
  @tailrec
  def go(from: A, acc: C): C = {
    if (continue(from))
      go(next(from), compute(acc, from))

sderosiaux

// From haskell "Monoidal Parsing—Edward Kmett"

import cats.implicits._

final case class Balance(l: Int, r: Int)
object Balance {
  val EMPTY = Balance(0, 0)
  val LEFT = Balance(0, 1)
  val RIGHT = Balance(1, 0)
}

sderosiaux

 trait Toto[F[_]] {
    def hello[A, B](fa: F[A])(f: A => B): F[B]
}

implicit val Fun = new Toto[List] {
  override def hello[A, B](fa: List[A])(f: A => B): List[B] = fa.map(f)
}

implicit class FunImplicits[F[_], A](t: F[A]) {
  def !!!![B](f: A => B)(implicit toto: Toto[F]): F[B] = toto.hello(t)(f)

sderosiaux

  def value(i: List[Int]): Trampoline[List[Int]] = i match {
    case Nil => Free.pure(Nil)
    case head :: tail => Free.defer(value(tail)).flatMap { l => Free.pure(head + 1 :: l) }
  }

  import cats.implicits._
  println(value((1 to 10).toList).run) // List(2, 3, 4, 5, 6, 7, 8, 9, 10, 11)

sderosiaux

for i in 1 2 3 4 5; do nslookup -q=TXT "_cloud-netblocks$i.googleusercontent.com" 8.8.8.8 | tr ' ' '\n' | grep ip4 | cut -d':' -f2 ; done

sderosiaux

  val builder = new KStreamBuilder
  val textLines = builder.stream[String, String]("TextLinesTopic")
  val pattern = Pattern.compile("\\W+")
  val wordCounts: KTable[String, String] = textLines
      .flatMapValues(line => pattern.split(line.toLowerCase).toIterable.asJava)
      .groupBy { case (_, word: String) => word }
      .count("Counts")
      .mapValues(_.toString)
      .to(Serdes.String, Serdes.String, "WordsWithCountsTopic")