randerzander

USER='admin'
PASS='admin'
CLUSTER='dev'
HOST=$(hostname -f):8080

function start(){
  curl -u $USER:$PASS -i -H 'X-Requested-By: ambari' -X PUT -d \
    '{"RequestInfo": {"context" :"Start '"$1"' via REST"}, "Body": {"ServiceInfo": {"state": "STARTED"}}}' \
    http://$HOST/api/v1/clusters/$CLUSTER/services/$1
}

randerzander

wget http://dagobah.ftphosting.net/yum/smartfile.repo -O /etc/yum.repos.d/smartfile.repo
mv smartfile.repo /etc/yum.repos.d/
wget http://dl.fedoraproject.org/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm
wget http://rpms.famillecollet.com/enterprise/remi-release-5.rpm
sudo rpm -Uvh remi-release-5*.rpm epel-release-5*.rpm
yum install -y libffi python26

randerzander

# Git clone, build, and copy the necessary MongoDB jars to your worker node $HADOOP_HOME/lib directories
cd ~/
git clone https://github.com/mongodb/mongo-hadoop
cd mongo-hadoop
./gradlew jar 
sudo cp build/libs/* /usr/lib/hadoop/lib
sudo cp core/build/libs/* /usr/lib/hadoop/lib
sudo cp hive/build/libs/* /usr/lib/hadoop/lib

cd ~/

randerzander

set number
set autoindent
set smartindent
set expandtab
set shiftwidth=2
set tabstop=2

randerzander

# Put a list of URLs in a file, inputs.txt
echo ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/daily/by_year/2009.csv.gz > input.txt
echo ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/daily/by_year/2010.csv.gz >> input.txt

# Trick to force MapReduce to treat each line in inputs.txt as a single map task
mkdir input
cd input
# I used 1 line per file - tweak for smaller downloads & fewer map tasks
split -l 1 ../input.txt
hadoop fs -put input .

randerzander

set -eu

FILE=$1
HEADER_LINE_NUM=$2
DELIM=$3
TABLE_NAME=$4
LOCATION=$5

# This script assumes columns are all strings. Edit the DDL file after running the script and change column types at will.

randerzander

DIR=$1

cd $DIR
for file in *
do
  tar -czvf $file.tgz $file/
done

randerzander

import jaydebeapi
conn = jaydebeapi.connect('org.apache.phoenix.jdbc.PhoenixDriver', \
                           ['jdbc:phoenix:my_zk_server:2181:/hbase-unsecure', '', ''], \
                           '/usr/hdp/current/phoenix-client/phoenix-client.jar')
curs = conn.cursor()
curs.execute('select * from WEB_STAT limit 1')
curs.fetchall()

randerzander

# Configure Flume agent (/etc/flume/conf/flume.conf) to receive syslog messages 
agent.sources=syslogsource-1
agent.channels=mem-channel-1
agent.sinks=kafka-sink-1

agent.sources.syslogsource-1.type=syslogtcp
agent.sources.syslogsource-1.port=13073
agent.sources.syslogsource-1.host=0.0.0.0
agent.sources.syslogsource-1.channels=mem-channel-1

randerzander

# See https://pip.pypa.io/en/latest/installing.html for how to install pip, then:
# pip install simple-salesforce
from simple_salesforce import Salesforce
import string

# To get your SFDC token, see https://help.salesforce.com/apex/HTViewHelpDoc?id=user_security_token.htm
# salesforce_login.txt should contain the following, one per line:
# Your Name
# your_email@sfdc_account.com
# your_password