ioleo

#!/bin/sh

# Pretty yellow message in a box
# Usage:
#   yell "My message"

yell() {
    dash=$(printf '=%.0s' $(seq 1 ${#1}))
    echo $(echo -e "\e[33m.=$dash=.\e[0m" | sed 's/=/-/g')
    echo -e "\e[33m| $1 |\e[0m"

ioleo

import org.apache.spark.sql.SparkSession

object Main extends App {
  val spark = SparkSession.builder().master("local[*]").getOrCreate()
  spark.experimental.extraOptimizations = Seq(MyRule)
  import spark.implicits._

  val numbersDS = Seq(0,1,2,3,4,5,6,7,8,9).toDS()
  println(numbersDS.map(_+2).explain(true))

ioleo

import scala.concurrent.{Await, Future}
import scala.concurrent.duration.Duration
import scala.concurrent.ExecutionContext.Implicits.global

object Main extends App {
  val seq = Future.sequence(Seq(
    Future { Thread.sleep(7000); println("doing 1"); 1 },
    Future { println("doing 2"); 2 },
    Future { Thread.sleep(3000); println("doing 3"); 3 },
    Future { println("doing 4"); 4 },

ioleo

// this is a disabled version of CassandraAsyncContext which does not connect to Cassandra DB on init

package loostro.scala.gist

import collection.JavaConverters._
import com.datastax.driver.core.Cluster
import com.datastax.driver.core.Configuration.Builder
import io.getquill.{CassandraAsyncContext, SnakeCase}
import java.net.InetSocketAddress
import org.scalatest.{FlatSpec, Matchers}

ioleo

package org.apache.spark.sql

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.catalyst.CatalystConf
import org.apache.spark.sql.catalyst.analysis.Analyzer
import org.apache.spark.sql.catalyst.catalog.SessionCatalog
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.internal.{SQLConf, SessionState}

ioleo

Advanced Functional Programming with Scala - Notes

Copyright © 2017 Fantasyland Institute of Learning. All rights reserved.

1. Mastering Functions

A function is a mapping from one set, called a domain, to another set, called the codomain. A function associates every element in the domain with exactly one element in the codomain. In Scala, both domain and codomain are types.

val square : Int => Int = x => x * x

ioleo

object OneWayImplicits extends App {

  //outside implicits visible, but T is not visible inside
  case class Outwards[T](value: T) extends AnyVal

  //outside implicits invisible, T is visible inside
  case class Inwards[T](value: T) extends AnyVal

  implicit def outwardsFromT[T](implicit t: T): Outwards[T] = Outwards(t)
  implicit def tFromInwards[T](implicit inw: Inwards[T]): T = inw.value

ioleo

object workspace {

  implicit class DefinedValuesMapExtension[K >: Null, V](map: Map[K, Option[V]]) {
    def toDefinedValuesMap: Map[K, V] = map.collect { case (key, value) if value.isDefined => (key, value.get) }
  }
  

  val mapWithOptionalValues: Map[String, Option[Int]] = Map(
    "foo" -> Some(20),
    "bar" -> None,

ioleo

(function (undefined) {
    /**
     * moment.easter
     * Source: https://github.com/zaygraveyard/moment-easter
     * License: MIT
     */
    moment.easter = function Easter20ops(year) {
        var a = (year / 100 | 0) * 1483 - (year / 400 | 0) * 2225 + 2613;
        var b = ((year % 19 * 3510 + (a / 25 | 0) * 319) / 330 | 0) % 29;
        var c = 148 - b - ((year * 5 / 4 | 0) + a - b) % 7;

ioleo

Spark-a-thon - Development Activities

Tuesday, September 12, 2017

Sparkathon - Developing Spark Structured Streaming Apps in Scala

Spark SQL

1. Multiple groupBy or groupByKey aggregations in a batch structured query