hellertime

import scala.collection.JavaConversions._

import cascading.scheme.Scheme
import cascading.tap.SinkMode
import cascading.tuple.Fields
import com.twitter.scalding.{FixedPathSource, HadoopSchemeInstance, SchemedSource}
import org.apache.hadoop.mapred.{JobConf, OutputCollector, RecordReader}

trait HiveScheme extends SchemedSource {
  // cascading-hive Schemes take two arrays as arguments

hellertime

Exploring the Typeclassopedia. First up: Functors.

A Functor is a type which satisfies the following laws:

fmap id = id
fmap (f . g) = fmap f . fmap g

Its declaration is:

class Functor f where

hellertime

package com.a.r.ingest.scalding

import scala.collection.JavaConversions._

import _root_.parquet.cascading.ParquetTupleScheme
import cascading.scheme.Scheme
import cascading.tap.SinkMode
import cascading.tuple.Fields
import com.twitter.scalding.FixedPathSource
import com.twitter.scalding.HadoopSchemeInstance

hellertime

$ git status -s
warning: skipping rename detection, detected duplicate destination 'src/main/java/com/a/r/s2s/eventbus/EventBus.java'
 D src/main
D  src/main/java/com/a/r/s2s/Flags.java
A  src/main/java/com/a/r/s2s/Main.java
A  src/main/java/com/a/r/s2s/Stream2StoreConfig.java
D  src/main/java/com/a/r/s2s/Stream2StoreCycler.java
D  src/main/java/com/a/r/s2s/Stream2StoreEvent.java
D  src/main/java/com/a/r/s2s/Stream2StoreExecutor.java
A  src/main/java/com/a/r/s2s/Stream2StoreScheduler.java

hellertime

> import Distribution.Simple

This is an example Setup.hs for a haskell program making use of HaskellDB.

> import System.Directory ( removeFile, removeDirectoryRecursive )

HaskellDB needs to have the structure of all database tables laid out as data. 

> import Database.HaskellDB
> import Database.HaskellDB.DBSpec

hellertime

> import Control.Exception ( handle, IOException )

here is a specialize handle which deals with IOExceptions

> handleIO :: (IOException -> IO a) -> IO a -> IO a
> handleIO = handle

Without this added signature you will get type-check errors like this:

    Ambiguous type variable `e' in the constraint:

hellertime

import sys
# requires Python 2.5
assert sys.version_info[1] >= 5, "Requires Python Coroutines. See PEP 0342"

def pushbackiter(iterator):
    """converts 'iterator' into a coroutine, with limited pushback

    >>> it = pushbackiter([1,2,3,4,5])
    >>> it.next()
    1

hellertime

intervals :: (Eq a, Enum a) => [a] -> [(a, Maybe a)]
intervals []     = []
intervals (x:[]) = [(x, Nothing)]
intervals (x:xs) = (x, lastSuccessor x x xs) : intervals (dropSuccessors x xs)
  where
    lastSuccessor :: (Eq a, Enum a) => a -> a -> [a] -> Maybe a
    lastSuccessor l x []
        | l == x                  = Nothing
        | otherwise               = Just x
    lastSuccessor l x (y:ys)

hellertime

#!/bin/sh
# Needed to rename a directory chock full o' files (don't ask).
# First thought?

find . -type f -exec rename 's/(\d+).(foo|bar|baz).(\d+).txt/$1.$3.$2.txt/' +

# Result? Horrible! 
# Not sure what was going on, but between find batching the arguments to rename, 
# and rename pausing after each batch, run-time was approaching 4 hours+!
#

hellertime

AH_VERBATIM([PORTABLE_BYTE_SWAP_FUNCTIONS],
[/* Standardize on a byte-swap function naming */
#if defined(__linux__)
#  include <endian.h>
#elif defined(__FreeBSD__) || defined(__NetBSD__)
#  include <sys/endian.h>
#elif defined(__OpenBSD__)
#  include <sys/types.h>
#  define be16toh(x) betoh16(x)
#  define be32toh(x) betoh32(x)