stettix

Things I believe

This is a collection of the things I believe about software development. I have worked for years building backend and data processing systems, so read the below within that context.

Agree? Disagree? Feel free to let me know at @JanStette. See also my blog at www.janvsmachine.net.

Fundamentals

Keep it simple, stupid. You ain't gonna need it.

stettix

package backfiller

import aws.AWS
import cats.effect.{ExitCode, IO, IOApp}
import com.amazonaws.AmazonServiceException
import com.amazonaws.retry.RetryUtils
import com.gu.emr.ClusterManager.ClusterID
import com.gu.emr.EmrClusterManager
import com.gu.emr.model.{ClusterDefinition, EmrStep, RunConfiguration}
import fs2.{Chunk, Pure, Stream}

stettix

  import spark.implicits._

  // Input data

  val identity =
    spark
      .table("clean.identity")
      .select("id", "account_created", "last_active_date", "primary_email_address")
      .cache()

stettix

  def eval(expr: Expr, env: Env): Expr = expr match {
    // ...
    case "cons" :: arg1 :: arg2 :: Nil => eval(arg2, env) match {
      case l: List[Expr] => eval(arg1, env) :: l
      case arg => throw new IllegalArgumentException(s"Second argument to 'cons' should be list, was: '$arg'")
    }
    case "cons" :: args => throw new IllegalArgumentException(s"Invalid arguments to 'cons': $args")
    case "null?" :: head :: Nil => eval(head, env) == List()
    case "null?" :: _ => throw new IllegalArgumentException("Expected exactly one argument for 'null?'")
    // ...