krishnanraman

/////////// INPUT PIPE = props.txt, Columns: date, displaylocation, engagements ///////////
2/1 1 10
2/1 1 20
2/1 2 30
2/1 2 10
2/1 3 20
2/1 4 10
2/2 1 10
2/2 2 20
2/2 2 15

krishnanraman

// And by the way, please feel free to contribute, correct errors, etc!

trait Functor[F[_]] {
    def fmap[A, B](f: A => B): F[A] => F[B]
}

object Functor {
  val ListFunctor: Functor[List] =
    new Functor[List] {
        def fmap[A, B](f: A => B) =

krishnanraman

trait Abelian

case class Zn(order:Int, zero:Int) extends Abelian {
     def identity = zero
     def size = order
     def elements =  (1 to order).toSeq
     def cayley = {
          Vector.tabulate(order,order)((x,y)=> {
               val idx = math.abs(zero - (x+1)) // difference between x & zero
               val timesToShift = if ((x+1)>=zero) idx else (order-idx)

krishnanraman

import com.backtype.hadoop.pail.PailStructure
import java.util.{ List => JList }
import scala.collection.JavaConverters._
import com.twitter.scalding._
import com.twitter.scalding.commons.source.{PailSource,CodecPailStructure}
import com.twitter.bijection.{NumericInjections, Injection}


class PailTest2(args : Args) extends Job(args) {
  val pipe = IterableSource((1 to 100), "src").read

krishnanraman

import com.backtype.hadoop.pail.PailStructure
import java.util.{ List => JList }
import scala.collection.JavaConverters._
import com.twitter.scalding._
import com.twitter.scalding.commons.source.{PailSource,CodecPailStructure}
import com.twitter.bijection.{NumericInjections, Injection}

class PailTest2Write(args : Args) extends Job(args) {
  args("io") match {
    case "read" => readjob

krishnanraman

Pail example: 

Writejob: Partition numbers [1..100] into two directories - belowfifty & abovefifty. 
Further, create 7 subdirectories under each, based on number mod 7. 
So a number like 62 would end up in the location "abovefifty/6".

Readjob: Read the subdirectories "belowfifty/3" & "abovefifty/0"

RESULTS:
$ tree pailtest

krishnanraman

import com.twitter.scalding._

class HistogramTest(args : Args) extends Job(args) {
  IterableSource(List((1,2),(3,4),(5,6),(7,8)), List('a,'b))
  .read
  .map(('a,'b)->('c)){
    x:(Int,Int) =>
    x._1+x._2
  }.write(Tsv("data/histo_test"))
}

krishnanraman

import com.twitter.scalding._
import com.twitter.scalding.mathematics.Histogram
import util.Random

class HistogramTest(args : Args) extends Job(args) {

  def cdf2pdf(cdf:Map[Double,Double], keys:List[Double], size:Int):Map[Double,Double] = {
    var m = Map[Double,Double]()
    keys.foldLeft((m, 0.0d))((a,b) =>{
      val myval = cdf(b)*size

krishnanraman

import com.twitter.scalding._
import util.Random

// Produce a histogram with 10 bins, of n numbers with Gaussian distribution N(5,1) ~ mean 5, stdev 1.
class HistogramTest(args : Args) extends Job(args) {

  def l2t10(x:List[Int]) = Tuple10(x(0), x(1),x(2),x(3), x(4),x(5),x(6), x(7),x(8),x(9))

  val tuples = (1 to args("n").toInt).map( x=> Random.nextGaussian + 5)
  val bins = 10

krishnanraman

/*
@author: Krishnan Raman
usage: GeoToZipCode.convert( latlng )
*/
import scala.io.Source

object GeoToZipcode {
  lazy val zip = {
    val lines = Source.fromFile("zipcodes.csv").getLines
    lines.next // skip header