jbenninghoff

// Use Gists to store code you would like to remember later on
console.log(window); // log the "window" object to the console

jbenninghoff

hduser@master:~$ cat wordcount.pig
A = load '/user/jbenninghoff/somefile.txt';
B = foreach A generate flatten(TOKENIZE((chararray)$0)) as word;
C = filter B by word matches '\\w+';
D = group C by word;
E = foreach D generate COUNT(C), group;
store E into '/user/jbenninghoff/somefileWordcount';

jbenninghoff

root@cent01 redhat 4C 02:43pm# sed -n '/^#/d;/@/d;s/\(^.*\): \(.*$\)/[\1]\n\2\n/p' /etc/clustershell/groups | sed '/\[.*\]/s/-/:/;/.* .*/s/ /\n/'
[all]
cent[01:05]

[zk]
cent[01:03]

[cldb]
cent01
cent02

jbenninghoff

#!/bin/bash

/opt/mapr/spark/spark-1.2.1/bin/spark-submit --master yarn-client \
  --class org.apache.spark.examples.terasort.TeraGen \
  --name 'TeraGen' \
  --conf 'mapreduce.terasort.num.partitions=5' \
  --executor-cores 30 \
  --executor-memory 7G \
  --num-executors 9 \
  terasort-project_2.10-1.0.jar 50G /user/$USER/spark-terasort

jbenninghoff

/*
 * Compile and run with:
 *  javac -cp $(hbase classpath) TestHBase.java 
 *  java -cp .:$(hbase classpath) TestHBase
 */
import java.net.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;

jbenninghoff

#!/bin/bash

echo 'Script not ready for execution.  Copy and paste line by line into a shell instead'
echo 'Assumes clush installed and /etc/hosts propagated to all nodes'
exit 1

#Configure edge node as MapR client
vi /etc/yum.repos.d/maprtech.repo # We should have rpm to install+enable like EPEL rpm
yum clean all
#Insure iptables (firewall) is off and disabled everywhere

jbenninghoff

#!/bin/bash
# jbenninghoff 2013-Sep-13  vi: set ai et sw=3 tabstop=3:

# Assumes MapR YCSB branch to handle large tables:  https://github.com/mapr/YCSB
# Assumes MapR HBase client software installed.  Can be an edge/gateway node

export HBASE_CLASSPATH=core/lib/core-0.1.4.jar:hbase-binding/lib/hbase-binding-0.1.4.jar
table=/benchmarks/usertable #YCSB uses table named 'usertable' by default
thrds=4
count=$[100*1000*1000] #table row count

jbenninghoff

#!/bin/bash
#jbenninghoff 2015-Dec-28  vi: set ai et sw=3 tabstop=3 retab:

: << '--BLOCK-COMMENT--'
Bash idioms template
Save as ~/.vim/templates/sh
Above requires vim templates plugin: https://github.com/ap/vim-templates
Useful site for lots of Bash info: http://wiki.bash-hackers.org/
--BLOCK-COMMENT--

jbenninghoff

#!/bin/bash

umount /home
lsblk -P /dev/sdb | grep -o MOUNTPOINT.*
lvremove -f vg_$(hostname -s|tr A-Z a-z)/lv_home
parted /dev/sdb -- rm 1
grep home /etc/fstab
sed -i.bak '/home/d' /etc/fstab
vgreduce -f vg_$(hostname -s|tr A-Z a-z) --removemissing
vgreduce -f vg_${HOSTNAME,,} --removemissing

jbenninghoff

Fix list from cluster-audit.sh findings:

Push mapr repos from node1 to all others.
Yum install dstat jq nmap nc tmux tuned vim xml2 zsh
Disable /etc/selinux/config and setenforce Permissive
chkconfig iptables off
echo 'vm.swappiness = 1' >> /etc/sysctl.conf
Not in tmpwatch: /tmp/hadoop-mapr/nm-local-dir
New hostname in /etc/sysconfig/network
Add all hosts to /etc/hosts