j14159

open Ppxlib

exception No_match
   
let assert_match_ext =
  Extension.declare
    "ppx_assert_match.assert_match"
    Extension.Context.Expression
    Ast_pattern.(ppat __ __)
    (fun ~loc ~path:_ patt guard ->

j14159

package main

type mesa struct {
	height int
	width  int
}

type position struct {
	x         int
	y         int

j14159

/*
 * Very naive approach to simulating a distributed cache using consistent hashing.  Based on the following:
 * http://thor.cs.ucsb.edu/~ravenben/papers/coreos/kll%2B97.pdf
 *
 * Initially a translation of:
 * http://weblogs.java.net/blog/tomwhite/archive/2007/11/consistent_hash.html
 *
 * And I also looked at this gist a bit from http://github.com/opyate towards the end as well:  
 * https://gist.github.com/1927001
 *

j14159

list_test_() ->
    [?_assertMatch({{t_list, t_float}, _},
                   top_typ_of("1.0 : []")),
     ?_assertMatch({{t_list, t_int}, _},
                   top_typ_of("1 : 2 : []")),
     ?_assertMatch({error, _}, top_typ_of("1 : 2.0 : []")),
     ?_assertMatch({{t_arrow, 
                     [{unbound, A, _}, {t_list, {unbound, A, _}}],
                     {t_list, {unbound, A, _}}}, _},
                   top_typ_of("f x y = x : y")),

j14159

"/hello/:name" get {
  request => Future {
    /*
    * we're in The Future here so you can do 
    * heavier stuff without blocking up other
    * things like the Netty handler.
    */
    val name = request.path(":name")
    Response.ok(s"Hello, ${name}")
  }

j14159

I'm putting this list together as a sort of reading plan for myself in order to learn more about general cluster scheduling/utilization and various ways of generically programming to them. Lists of direct links to PDFs here in the order I think makes some sense from skimming reference sections.

Happy to here of any additions that might be sensible.

The Basics

1. Google File System since everything references it and data locality is a thing.
2. Google MapReduce because it's one of the earlier well-known functional approaches to programming against a cluster.
3. Dryad for a more general (iterative?) programming model.
4. Quincy for a different take on scheduling.
5. [Delay Scheduling](h

j14159

/*
 * A more recent version of my S3N RDD.  This exists because I needed
 * a reliable way to distribute the fetching of S3 data using instance
 * credentials as well as a simple way to filter out the inputs that
 * I didn't want in the RDD.
 *
 * This version is more eager than the last one and also provides a
 * simple RDD that allows you to tag each line with information about
 * its partition/source.
 * 

j14159

#!/bin/sh

INSTANCE_ID="`curl http://169.254.169.254/latest/meta-data/instance-id`"
REGION="`curl http://169.254.169.254/latest/meta-data/placement/availability-zone | sed s'/[a-zA-Z]$//'`"
ZK_HOSTS="`aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" "Name=key,Values=mesos-master" --region $REGION --output=text | cut -f5`"

ulimit -n 200000

LD_LIBRARY_PATH=/usr/lib/jvm/java-7-openjdk-amd64/jre/lib/amd64/jamvm nohup /usr/local/sbin/mesos-slave --master=${ZK_HOSTS} --ip=0.0.0.0 --isolation=cgroups --no-switch_user --log_dir=/var/log/mesos &

j14159

(require 'package)
(add-to-list 'package-archives
         '("marmalade" . "http://marmalade-repo.org/packages/"))
(add-to-list 'package-archives
  '("melpa" . "http://melpa.milkbox.net/packages/") t)

(setq-default indent-tabs-mode nil)

(define-key global-map (kbd "C-c SPC") 'ace-jump-mode)
(define-key global-map (kbd "C-x g") 'magit-status)

j14159

#!/bin/bash

#
# WARNING:  This will wipe and encrypt the device given.  For Mesos workers,
# this is run on EVERY BOOT so you will constantly lose existing data.
#

# I have based this script on the following links:
# https://github.com/matthew-lucidchart/aws-ephemeral-mounts/blob/master/boot_luks.sh
# http://nineofclouds.blogspot.ca/2013/10/how-to-use-lvm-and-luks-with-ebs-volumes.html