gerigk

runners:
  emr:
    aws_access_key_id: youraccountid
    #aws_region: us-west-1   your region. us east by default
    aws_secret_access_key: yoursecretkey
    bootstrap_actions:
    # probably this is a good idea
    - s3://elasticmapreduce/bootstrap-actions/configurations/latest/memory-intensive
    # we disable this since it is run before our shell script and installs mrjob for python 2.6
    bootstrap_mrjob: False

gerigk

#!/bin/bash
###################
#configuration here
####################
bucketname="your_bucket_name"
##########################
cd /home/hadoop
#first we set two vars...I had errors without this
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
export LD_RUN_PATH=/usr/local/lib:$LD_RUN_PATH

gerigk

bucketname="yourbucketname"
cd /home/hadoop
wget http://www.netlib.org/lapack/lapack-3.4.1.tgz
wget http://downloads.sourceforge.net/project/math-atlas/Developer%20%28unstable%29/3.9.76/atlas3.9.76.tar.bz2
tar -vxf atlas3.9.76.tar.bz2
cd ATLAS
mkdir build
cd build
################################## -t 2 means 2 threads. depending on the ec2 instance you can choose more threads 14
###  V 448 means SSE1/2/3 support. A14 means x86SSE364SSE2 architecture. check the documentation for more information

gerigk

#!/bin/bash
###################
#configuration here
####################
bucketname="my_bucket_name"
##########################

cd /home/hadoop
#first we set two vars...I had errors without this
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH

gerigk

import psycopg2
import os
import sys
sys.path.append(os.path.abspath('../includes'))
import dbLoader
from datetime import datetime


class ReadFaker:
    """

gerigk

@classmethod
def from_dict(cls, data, orient='columns', dtype=None):

        from collections import defaultdict

        orient = orient.lower()
        if orient == 'index':
            # TODO: this should be seriously cythonized
            new_data = defaultdict(dict)
            for index, s in data.iteritems():