Setup Erie via BP on CentOS6

setup_erie.sh

# One liner to install HDP Erie TP on a single node
# export host_count=1; curl -sSL https://gist.github.com/abajwa-hw/8930ff5eee82285930a4c2f697310729/raw | sudo -E sh


export ambari_services="ZOOKEEPER HDFS YARN MAPREDUCE2 HIVE PIG SLIDER TEZ"
export cluster_name=HDP
export ambari_password="admin"

#TP 6
export ambari_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/ambari/2.4.0.0-7/centos6/ambari.repo"
export hdp_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/2.5.0.0-7/centos6"

#TP 4
#export ambari_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/ambari/2.4.0.0-5/centos6/ambari.repo"
#export hdp_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/2.5.0.0-5/centos6"

#TP3
#export ambari_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/ambari/2.4.0.0-4/centos6/ambari.repo"
#export hdp_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/2.5.0.0-4/centos6"

#TP #2 
#export ambari_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/ambari/2.4.0.0-2/centos6/ambari.repo"
#export hdp_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/2.5.0.0-2/centos6"


# Before increasing host_count, ensure you have installed/registered ambari-agents on other nodes. You can do this by running below on those nodes (replace myambariserver.hortonworks.com with the FQDN of the ambari-server node)
# export ambari_server=myambariserver.hortonworks.com ; export ambari_version=2.4.0.0; export ambari_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/ambari/2.4.0.0-5/centos6/ambari.repo"
# curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh
export host_count=${host_count:-1}


rm -rf ~/ambari-bootstrap

set -e

#install ambari server 
yum install -y git
git clone https://github.com/seanorama/ambari-bootstrap.git
export install_ambari_server=true
export ambari_version=2.4.0.0

~/ambari-bootstrap/ambari-bootstrap.sh
echo "Sleeping until Ambari server is fully up"
sleep 45

cd ~

#alias curl="curl -L -H X-Requested-By:blah -u admin:admin"

# register utils repo
tee /tmp/repo.json > /dev/null << EOF
{
  "Repositories": {
    "base_url": "http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.21/repos/centos6",
    "verify_base_url": true
  }
}
EOF

    
url="http://localhost:8080/api/v1/stacks/HDP/versions/2.5/operating_systems/redhat6/repositories/HDP-UTILS-1.1.0.21"
curl -L -H X-Requested-By:blah -u admin:admin -X PUT "${url}" -d @/tmp/repo.json
curl -L -H X-Requested-By:blah -u admin:admin "${url}"
rm -f /tmp/repo.json

# register HDP repo

tee /tmp/repo.json > /dev/null << EOF
{
  "Repositories": {
    "base_url": "$hdp_repo",
    "verify_base_url": true
  }
}
EOF

url="http://localhost:8080/api/v1/stacks/HDP/versions/2.5/operating_systems/redhat6/repositories/HDP-2.5"
curl -L -H X-Requested-By:blah -u admin:admin -X PUT "${url}" -d @/tmp/repo.json
curl -L -H X-Requested-By:blah -u admin:admin "${url}"
rm -f /tmp/repo.json

cd ~/ambari-bootstrap/deploy/

# disable bootstrap's default configs

cat << EOF > configuration-custom.json
{
  "configurations" : {
    "core-site": {
        "hadoop.proxyuser.HTTP.groups" : "users,hadoop-users",
        "hadoop.proxyuser.HTTP.hosts" : "*",
        "hadoop.proxyuser.hbase.groups" : "users,hadoop-users",
        "hadoop.proxyuser.hbase.hosts" : "*",
        "hadoop.proxyuser.hcat.groups" : "users,hadoop-users",
        "hadoop.proxyuser.hcat.hosts" : "*",
        "hadoop.proxyuser.hive.groups" : "users,hadoop-users",
        "hadoop.proxyuser.hive.hosts" : "*",
        "hadoop.proxyuser.knox.groups" : "users,hadoop-users",
        "hadoop.proxyuser.knox.hosts" : "*",
        "hadoop.proxyuser.oozie.groups" : "users",
        "hadoop.proxyuser.oozie.hosts" : "*",
        "hadoop.proxyuser.root.groups" : "*",
        "hadoop.proxyuser.root.hosts" : "*",
        "fs.trash.interval": "4320"
    },
    "hive-interactive-env": {
        "enable_hive_interactive": "true",
        "llap_queue_capacity": "75"
    },
    "hive-interactive-site": {
      "hive.server2.tez.default.queues" : "llap"
    },
    "yarn-site": {
        "yarn.acl.enable" : "true"
    },
    "hdfs-site": {
      "dfs.namenode.safemode.threshold-pct": "0.99"
    },
    "hive-site": {
        "hive.exec.compress.output": "true",
        "hive.merge.mapfiles": "true",
        "hive.server2.tez.initialize.default.sessions": "true"
    },
    "mapred-site": {
        "mapreduce.job.reduce.slowstart.completedmaps": "0.7",
        "mapreduce.map.output.compress": "true",
        "mapreduce.output.fileoutputformat.compress": "true"
    }
  }
}
EOF

ambari-server restart
sleep 10


#deploy cluster
export ambari_stack_name=HDP
export ambari_stack_version=2.5
export recommendation_strategy="ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES"

~/ambari-bootstrap/deploy/deploy-recommended-cluster.bash

sleep 5

#wait until cluster deployed
source ~/ambari-bootstrap/extras/ambari_functions.sh
ambari_configs
ambari_wait_request_complete 1

sleep 10

#create user dirs
sudo -u hdfs hdfs dfs -mkdir /user/admin
sudo -u hdfs hdfs dfs -chown admin /user/admin
sudo -u hdfs hdfs dfs -mkdir /user/root
sudo -u hdfs hdfs dfs -chown root /user/root
sudo -u hdfs hdfs dfs -mkdir /user/anonymous
sudo -u hdfs hdfs dfs -chown root /user/anonymous

#import sample data
echo "import data into hive"
cd /tmp
wget https://raw.githubusercontent.com/abajwa-hw/security-workshops/master/data/sample_07.csv

url="jdbc:hive2://$(hostname -f):2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2-hive2"

beeline -u $url -e "CREATE TABLE sample_07 (
code string ,
description string ,  
total_emp int ,  
salary int )
ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TextFile; "

beeline -u $url -e "load data local inpath '/tmp/sample_07.csv' into table sample_07;"

#Run test beeline queries against Interactive Hive (hive LLAP)
beeline -u $url -e "select count(*) from sample_07 where salary > 10000;"
beeline -u $url -e "select count(*) from sample_07 where salary > 100000;"