Deploy HDP plus syncsort

deploy_hdp_syncsort.sh

#To run, first copy DMX ambari package to /tmp
#   scp -i ~/.ssh/field.pem ~/Downloads/dmexpress-9.2-el7.ambari-service_en.bin centos@172.26.194.217:/tmp
#then export any variables and then execute below:
#   curl -sSL https://gist.github.com/abajwa-hw/a4bec6c5fe7545f3515893437746fd38/raw | sudo -E sh 

#For multinode, prep the other hosts first by running below (replace ambari.xxx.com)
#   sudo rpm -Uvh http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm
#   export ambari_version="2.6.1.0";export ambari_server="ambari.xxx.com";curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh

#set -x

export ambari_password=${ambari_password:-BadPass#1}     #Ambari password
export host_count=${host_count:-1}   #choose number of nodes
export ambari_services=${ambari_services:-HDFS HIVE PIG MAPREDUCE2 TEZ YARN ZOOKEEPER DMXh}  #AMBARI_METRICS can be added post-install
export hdp_ver=${hdp_ver:-2.6}
export ambari_version="${ambari_version:-2.6.1.0}"
export dmx_ambari_service_package="/tmp/dmexpress-9.2-el7.ambari-service_en.bin"

sudo yum install -y git python-argparse
cd ~
git clone https://github.com/seanorama/ambari-bootstrap.git

#remove unneeded repos for some AMIs
if [ -f /etc/yum.repos.d/zfs.repo ]; then
  rm -f /etc/yum.repos.d/zfs.repo
fi

if [ -f /etc/yum.repos.d/lustre.repo ]; then
  rm -f /etc/yum.repos.d/lustre.repo
fi  


#install Ambari
export install_ambari_server=true
#curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh
~/ambari-bootstrap/ambari-bootstrap.sh

echo "Waiting 30s for Ambari to come up..."
sleep 30

echo "install MySQL community and connector"
sudo yum -y install mysql-connector-java
#install MySql community rpm
sudo rpm -Uvh http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm

sudo ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar


echo "Changing Ambari password..." 
sudo curl -iv -u admin:admin -H "X-Requested-By: blah" -X PUT -d "{ \"Users\": { \"user_name\": \"admin\", \"old_password\": \"admin\", \"password\": \"${ambari_password}\" }}" http://localhost:8080/api/v1/users/admin


##add syncsort service and restart Ambari
cat << EOF > input.txt


A


y
EOF

chmod +x ${dmx_ambari_service_package}
sudo ${dmx_ambari_service_package} < input.txt



## deploy cluster
echo "Setting recommendation strategy..."
export ambari_stack_version=${hdp_ver}
export recommendation_strategy="ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES"

echo "Generating BP and deploying cluster..."
cd ~/ambari-bootstrap/deploy
cat << EOF > configuration-custom.json
{
  "configurations" : {
    "core-site": {
        "hadoop.proxyuser.root.users" : "admin",
        "fs.trash.interval": "4320"
    },
    "hdfs-site": {
      "dfs.replication": "1",
      "dfs.namenode.safemode.threshold-pct": "0.99"
    },
    "hive-site": {
        "hive.server2.transport.mode" : "binary"
    }
 }
}
EOF

./deploy-recommended-cluster.bash

sleep 20
echo "Waiting for cluster to deploy..."
ambari_pass=${ambari_password} source ~/ambari-bootstrap/extras/ambari_functions.sh
ambari_configs
ambari_wait_request_complete 1
sleep 5
        
echo "generating test MySql db"
cd ~
git clone https://github.com/datacharmer/test_db
cd test_db/
sudo mysql < employees.sql