jbenninghoff

// Use Gists to store code you would like to remember later on
console.log(window); // log the "window" object to the console

jbenninghoff

hduser@master:~$ cat wordcount.pig
A = load '/user/jbenninghoff/somefile.txt';
B = foreach A generate flatten(TOKENIZE((chararray)$0)) as word;
C = filter B by word matches '\\w+';
D = group C by word;
E = foreach D generate COUNT(C), group;
store E into '/user/jbenninghoff/somefileWordcount';

jbenninghoff

root@cent01 redhat 4C 02:43pm# sed -n '/^#/d;/@/d;s/\(^.*\): \(.*$\)/[\1]\n\2\n/p' /etc/clustershell/groups | sed '/\[.*\]/s/-/:/;/.* .*/s/ /\n/'
[all]
cent[01:05]

[zk]
cent[01:03]

[cldb]
cent01
cent02

jbenninghoff

/*
 * Compile and run with:
 *  javac -cp $(hbase classpath) TestHBase.java 
 *  java -cp .:$(hbase classpath) TestHBase
 */
import java.net.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;

jbenninghoff

#!/bin/bash

echo 'Script not ready for execution.  Copy and paste line by line into a shell instead'
echo 'Assumes clush installed and /etc/hosts propagated to all nodes'
exit 1

#Configure edge node as MapR client
vi /etc/yum.repos.d/maprtech.repo # We should have rpm to install+enable like EPEL rpm
yum clean all
#Insure iptables (firewall) is off and disabled everywhere

jbenninghoff

#!/bin/bash
# jbenninghoff 2013-Sep-13  vi: set ai et sw=3 tabstop=3:

# Assumes MapR YCSB branch to handle large tables:  https://github.com/mapr/YCSB
# Assumes MapR HBase client software installed.  Can be an edge/gateway node

export HBASE_CLASSPATH=core/lib/core-0.1.4.jar:hbase-binding/lib/hbase-binding-0.1.4.jar
table=/benchmarks/usertable #YCSB uses table named 'usertable' by default
thrds=4
count=$[100*1000*1000] #table row count

jbenninghoff

Fix list from cluster-audit.sh findings:

Push mapr repos from node1 to all others.
Yum install dstat jq nmap nc tmux tuned vim xml2 zsh
Disable /etc/selinux/config and setenforce Permissive
chkconfig iptables off
echo 'vm.swappiness = 1' >> /etc/sysctl.conf
Not in tmpwatch: /tmp/hadoop-mapr/nm-local-dir
New hostname in /etc/sysconfig/network
Add all hosts to /etc/hosts

jbenninghoff

#!/bin/bash
# jbenninghoff 2015-Nov-13  vi: set ai et sw=3 tabstop=3 retab:

cnt=5 #cluster node count, does not include edge node
keyname=xxx #replace with your AWS Key Name

# Make sure your AWS firewall (security-group) does not block DNS (UDP)

# Launch AWS server(VM) to host Mesos Master, DNS, Marathon, Myriad host (edge node)
edgeID=$(aws ec2 run-instances --image-id ami-d2c924b2 --count 1 --instance-type m4.xlarge --key-name $keyname --security-group-ids sg-6730dd03 --subnet-id subnet-8deb24fa --block-device-mapping "DeviceName=/dev/sda1,Ebs={VolumeSize=300}" --query 'Instances[0].InstanceId')

jbenninghoff

FIO 4K vs 64K file size throughput.  64K and 512K file counts.  Loopback NFS mount on single node cluster.  fio and NFS/MFS on same machine.  16 threads in fio client.
 
rm smallFiles.*; fio --name=smallFiles --numjobs=16 --nrfiles=$[8*8] --filesize=4K --bs=4k --thread=1 --direct=1 --rw=write > fio-smallFiles-64x4K-16T-nfs-mfs.log
rm smallFiles.*; fio --name=smallFiles --numjobs=16 --nrfiles=$[8*8*8] --filesize=4K --bs=4k --thread=1 --direct=1 --rw=write > fio-smallFiles-512x4K-16T-nfs-mfs.log

rm smallFiles.*; fio --name=smallFiles --numjobs=16 --nrfiles=$[8*8*8] --filesize=64K --bs=4k --thread=1 --direct=1 --rw=write > fio-smallFiles-512x64K-16T-nfs-mfs.log
rm smallFiles.*; fio --name=smallFiles --numjobs=16 --nrfiles=$[8*8] --filesize=64K --bs=4k --thread=1 --direct=1 --rw=write > fio-smallFiles-64x64K-16T-nfs-mfs.log

==> fio-smallFiles-512x4K-16T-nfs-mfs.log <==

jbenninghoff

#!/bin/bash
#jbenninghoff 2015-Dec-28  vi: set ai et sw=3 tabstop=3 retab:

: << '--BLOCK-COMMENT--'
Bash idioms template
Save as ~/.vim/templates/sh
Above requires vim templates plugin: https://github.com/ap/vim-templates
Useful site for lots of Bash info: http://wiki.bash-hackers.org/
--BLOCK-COMMENT--