j14159

open Ppxlib

exception No_match
   
let assert_match_ext =
  Extension.declare
    "ppx_assert_match.assert_match"
    Extension.Context.Expression
    Ast_pattern.(ppat __ __)
    (fun ~loc ~path:_ patt guard ->

j14159

package main

type mesa struct {
	height int
	width  int
}

type position struct {
	x         int
	y         int

j14159

list_test_() ->
    [?_assertMatch({{t_list, t_float}, _},
                   top_typ_of("1.0 : []")),
     ?_assertMatch({{t_list, t_int}, _},
                   top_typ_of("1 : 2 : []")),
     ?_assertMatch({error, _}, top_typ_of("1 : 2.0 : []")),
     ?_assertMatch({{t_arrow, 
                     [{unbound, A, _}, {t_list, {unbound, A, _}}],
                     {t_list, {unbound, A, _}}}, _},
                   top_typ_of("f x y = x : y")),

j14159

I'm putting this list together as a sort of reading plan for myself in order to learn more about general cluster scheduling/utilization and various ways of generically programming to them. Lists of direct links to PDFs here in the order I think makes some sense from skimming reference sections.

Happy to here of any additions that might be sensible.

The Basics

1. Google File System since everything references it and data locality is a thing.
2. Google MapReduce because it's one of the earlier well-known functional approaches to programming against a cluster.
3. Dryad for a more general (iterative?) programming model.
4. Quincy for a different take on scheduling.
5. [Delay Scheduling](h

j14159

/*
 * A more recent version of my S3N RDD.  This exists because I needed
 * a reliable way to distribute the fetching of S3 data using instance
 * credentials as well as a simple way to filter out the inputs that
 * I didn't want in the RDD.
 *
 * This version is more eager than the last one and also provides a
 * simple RDD that allows you to tag each line with information about
 * its partition/source.
 * 

j14159

#!/bin/sh

INSTANCE_ID="`curl http://169.254.169.254/latest/meta-data/instance-id`"
REGION="`curl http://169.254.169.254/latest/meta-data/placement/availability-zone | sed s'/[a-zA-Z]$//'`"
ZK_HOSTS="`aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" "Name=key,Values=mesos-master" --region $REGION --output=text | cut -f5`"

ulimit -n 200000

LD_LIBRARY_PATH=/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/jamvm nohup /usr/local/sbin/mesos-slave --master=${ZK_HOSTS} --ip=0.0.0.0 --isolation=cgroups --no-switch_user --log_dir=/var/log/mesos &

j14159

(require 'package)
(add-to-list 'package-archives
         '("marmalade" . "http://marmalade-repo.org/packages/"))
(add-to-list 'package-archives
  '("melpa" . "http://melpa.milkbox.net/packages/") t)

(setq-default indent-tabs-mode nil)

(define-key global-map (kbd "C-c SPC") 'ace-jump-mode)
(define-key global-map (kbd "C-x g") 'magit-status)

j14159

#!/bin/bash

#
# WARNING:  This will wipe and encrypt the device given.  For Mesos workers,
# this is run on EVERY BOOT so you will constantly lose existing data.
#

# I have based this script on the following links:
# https://github.com/matthew-lucidchart/aws-ephemeral-mounts/blob/master/boot_luks.sh
# http://nineofclouds.blogspot.ca/2013/10/how-to-use-lvm-and-luks-with-ebs-volumes.html

j14159

val pool = new BoneCP(config)

val getConn = () => pool.getConnection()
val relConn = (c: Connection) => pool.releaseConnection(c)

class MyActor(get: () => Connection, rel: (c: Connection) => Unit) extends Actor {
  lazy val c = get()
  
  override def preRestart(why: Throwable, msg: Option[Any]): Unit = {
    rel(c)

j14159

trait PersonClient {
  // supply a router with a pool of PersonDao:
  val personPool: ActorRef

  // how long should we wait for a response from PersonDao:
  val timeoutInMillis: Long

  implicit val timeout = Timeout(timeoutInMillis millis)

  def addPerson(p: Person): Future[Int] =