abajwa-hw

###########################################################################################################################
##HDB on HDP sandbox setup script

###Pre-reqs: 
#- Laptop with at least 10-12 GB RAM (mine has 16 GB)
#- ISO image of Centos 6.7 or later downloaded from [here](http://isoredirect.centos.org/centos/6/isos/x86_64/). 
#  - In my case, I used CentOS-6.7-x86_64-bin-DVD1.iso.

##### Setup Centos 6.7 or later on VM
#- Start a CentOS VM using ISO 

abajwa-hw

#!/usr/local/bin/python

def post_request(url, body):
  import json, urllib2
  encoded_body = json.dumps(body)
  req = urllib2.Request(str(url), encoded_body)
  req.get_method = lambda: 'PUT'
  try:
    response = urllib2.urlopen(req, encoded_body).read()
  except urllib2.HTTPError, error:

abajwa-hw

{"paragraphs":[{"title":"Create HAWQ table and generate data series","text":"%psql.sql\ndrop table if exists tt;\ncreate table tt (i int);\ninsert into tt select generate_series(1,1000000);","dateUpdated":"Jun 7, 2016 7:12:59 PM","config":{"colWidth":12,"editorMode":"ace/mode/scala","title":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1465351928921_1500086996","id":"20160603-083343_921281900","result":{"code":"SUCCESS","type":"TABLE","msg":"Update Count\n0\n","comment":"","msgTable":[[{"value":"0"}]],"columnNames":[{"name":"Update Count","index":0,"aggr":"sum"}],"rows":[["0"]]},"dateCreated":"Jun 7, 2016 7:12:08 PM","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:863","dateFinished":"Jun 7, 2016 7:13:00 PM","dateStarted":"Jun 7, 2016 7:12:59 PM","focus":true},{"title":"Calculate average of subset of data","text":"%psql.sql\nselect avg(i) from tt where 

abajwa-hw

#  - Run the below script. You can customize the functionality by modifying the exports below. Otherwise to run the script with defaults, just run below one-liner and wait for 30 min:
# wget https://gist.github.com/abajwa-hw/ae026c63260ed6284b3d0333eac2c22d/raw -O setup_hdf.sh ; chmod +x setup_hdf.sh; export hdf_build=551; ./setup_hdf.sh ;

export ranger_user="${ranger_user:-rangeradmin}"
export ranger_pass="${ranger_pass:-BadPass#1}"
export nifi_cert_pass="${nifi_cert_pass:-BadPass#1}"
export hdf_build="${hdf_build:-579}"

read -p "Git Userid: " git_user
read -s -p "Git Password: " git_password

abajwa-hw

Automated deployment of fresh cluster via blueprints (using ambari-bootstrap)

• Note: You will now need to set the following env vars with your Cloudera paywall credentials: username paywall_password

• Bring up 4 VMs imaged with RHEL/CentOS 6.x (e.g. node1-4 in this case)

• On non-ambari nodes (e.g. nodes 2-4), install ambari-agents and point them to ambari node (e.g. node1 in this case)

export ambari_server=node1
export ambari_version=2.7.5.0
export username=myuser

abajwa-hw

• Create and distribute certs on HDF cluster

#download toolkit
wget https://hipchat.hortonworks.com/files/1/2055/bT1LbKB8SS26X9t/nifi-toolkit-1.0.0-SNAPSHOT-bin.zip

#create nifi certs dir under ambari-server resources dir
mkdir /var/lib/ambari-server/resources/host_scripts/nifi-certs

#generate certs using toolkit into ambari-server resources dir

abajwa-hw

# Sample script to deploy HDF via blueprint
# - Installs Ambari server/agents
# - Installs HDF mpack 
# - Uses ambari-bootstrap to generate blueprint based on stack advisor recommendation and starts cluster install
# - Optionally: installs KDC, sets up postgres for Ranger, allows customizations of config properties and number of Nifi nodes
#
# Usage: su as root and run below to invoke this script on a host where CentOS/RHEL has been freshly installed (do NOT run this on HDP sandbox!). You can customize the functionality by setting env vars e.g.
#  export host_count=3; export install_nifi_on_all_nodes=true; curl -sSL https://gist.github.com/abajwa-hw/ae4125c5154deac6713cdd25d2b83620/raw | sudo -E sh ;

# Note for multi-node install, you will need to install/register agents on other nodes first using below (replace <AMBARI_SERVER_HOSTNAME>)

abajwa-hw

# One liner to install HDP Erie TP on a single node
# export host_count=1; curl -sSL https://gist.github.com/abajwa-hw/8930ff5eee82285930a4c2f697310729/raw | sudo -E sh


export ambari_services="ZOOKEEPER HDFS YARN MAPREDUCE2 HIVE PIG SLIDER TEZ"
export cluster_name=HDP
export ambari_password="admin"

#TP 6
export ambari_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/ambari/2.4.0.0-7/centos6/ambari.repo"

abajwa-hw

# curl -sSL https://gist.github.com/abajwa-hw/f8b83e1c12abb1564531e00836b098fa/raw | sudo -E sh 
export host=$(hostname -f)
export realm=${realm:-HORTONWORKS.COM}
export domain=${domain:-hortonworks.com}
export kdcpassword=${kdcpassword:-BadPass#1}

set -e

sudo yum -y install krb5-server krb5-libs krb5-auth-dialog krb5-workstation

abajwa-hw

# Run this script on Ranger host to 
# 1. create users in format $host.openstacklocal@apache.nifi e.g. abajwa-hdf-qe-bp-1.openstacklocal@apache.nifi
# 2. create Ranger policies for above Nifi users:
#  a) read policy for /flow 
#  b) read/write policies for /proxy


export admin=${admin:-nifiadmin}
export cluster=${cluster:-HDF}
export hosts=${hosts:-myhost1 myhost2 myhost3}