Skip to content

Instantly share code, notes, and snippets.

@khalidr
khalidr / SparkSpec.scala
Last active November 6, 2023 17:55
Shared SparkSession for tests
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.my.tests.SparkSpec.sparkSession
import org.scalatest.flatspec.FixtureAnyFlatSpec
import org.scalatest.matchers.should.Matchers
import org.scalatest.Outcome
trait SparkSpec extends FixtureAnyFlatSpec with Matchers {
def parMergeSort(l:List[Int]):Future[List[Int]] = {
def merge(elements: List[Int]): Future[List[Int]] = {
elements match {
case ll if ll.size <= 1 ⇒ Future.successful(elements)
case ll ⇒
val middle = ll.size / 2
val (left, right) = ll.splitAt(middle)
val mLeft = Future(merge(left))
@khalidr
khalidr / SizeOnHeapEstimate.scala
Last active May 22, 2019 20:56
Get an estimate of the number of bytes an object takes up on heap
def size[T <: AnyRef](obj:T) = org.apache.spark.util.SizeEstimator.estimate(obj)
val myObj = new MyObj()
println(s"object size is ${size(myObj)}")
def fib1(i:Int) = {
//0,1,1,2,3,5,8,11
def loop(n1:Int,n2:Int0, idx:Int=1):Int = {
idx match {
case 0 => n2
case 1 => n1
case _ => loop(n2, n1+n2, idx-1)
}
}
loop(i)
@khalidr
khalidr / EmbeddedMongoSuite.scala
Created August 10, 2016 16:27
Use FlapDoodle to unit test Mongo
import de.flapdoodle.embed.mongo.config.{MongodConfigBuilder, RuntimeConfigBuilder}
import de.flapdoodle.embed.mongo.distribution.Version
import de.flapdoodle.embed.mongo.{Command, MongodStarter}
import de.flapdoodle.embed.process.config.io.ProcessOutput
import org.scalatest.{BeforeAndAfterAll, Suite}
import reactivemongo.api.MongoDriver
class EmbeddedMongoSuite extends Suite with BeforeAndAfterAll {
lazy val mongodExecutable = {
val runtimeConfig = new RuntimeConfigBuilder().defaults(Command.MongoD).processOutput(ProcessOutput.getDefaultInstanceSilent).build
@khalidr
khalidr / ErrorHandling.scala
Created April 8, 2016 16:05
ErrorHandling using AllCatch
//structural typing
def closingOrElse[T,CLOSEABLE <: {def close() : Unit}](resource: CLOSEABLE)(block: CLOSEABLE => T)(handler: Throwable => T) : T = {
allCatch.andFinally(ignoring(classOf[Throwable]){ resource.close() }){
allCatch.withApply{ e =>handler(e) }
{
block(resource)
}
}
}
@khalidr
khalidr / test.scala
Last active January 23, 2016 18:41
Complete With BadRequest (spray 1.3.3)
val r = {
path("foo"){
get{
complete(StatusCodes.BadRequest)
}
}
}
Get("/foo") ~> r ~> check{
println(response.status) // returns "400 Bad Request"
import spray.routing._
class UsersRouteSpec extends RouteSpec with UsersRoute with PlayJsonSupport with MockitoSugar{
//TODO: write some negative tests - particularly, failed credential update
it must "reject a get request if the session user is not authorized" in {
val fakeRoute = authorize(false){
complete{
@khalidr
khalidr / gist:38a2c2f18548afd8274f
Last active August 29, 2015 14:14
ApplicativeBuilder
case class Foo(f:String="",y:String,z:String)
object Foo{
type Val = Validated[Foo]
implicit val x = new Validator[Foo]{
def apply(v1: Foo): Validated[Foo] ={
import v1._
z.successNel[DomainValidationError] <*>(y.successNel[DomainValidationError] <*> (f.successNel[DomainValidationError] map (Foo.apply _).curried))
}