Setup HDF via blueprints

setup_hdf_blueprint.sh

# Sample script to deploy HDF via blueprint
# - Installs Ambari server/agents
# - Installs HDF mpack 
# - Uses ambari-bootstrap to generate blueprint based on stack advisor recommendation and starts cluster install
# - Optionally: installs KDC, sets up postgres for Ranger, allows customizations of config properties and number of Nifi nodes
#
# Usage: su as root and run below to invoke this script on a host where CentOS/RHEL has been freshly installed (do NOT run this on HDP sandbox!). You can customize the functionality by setting env vars e.g.
#  export host_count=3; export install_nifi_on_all_nodes=true; curl -sSL https://gist.github.com/abajwa-hw/ae4125c5154deac6713cdd25d2b83620/raw | sudo -E sh ;

# Note for multi-node install, you will need to install/register agents on other nodes first using below (replace <AMBARI_SERVER_HOSTNAME>)
# export ambari_server=<AMBARI_SERVER_HOSTNAME>; curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh ;
# e.g.
# export ambari_server=abajwa-hdf-qe-bp-1.openstacklocal; export ambari_version=2.4.1.0; curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh ;
# see for more details: https://community.hortonworks.com/articles/56849/automate-deployment-of-hdf-20-clusters-using-ambar.html

#clean previous setup
rm -rf ~/ambari-bootstrap
rm -f *_payload
rm -rf ~/hdf_ambari_mp
rm ~/apache-maven-3.0.5-bin.tar.gz

#remove unneeded repos
if [ -f /etc/yum.repos.d/zfs.repo ]; then
  rm -f /etc/yum.repos.d/zfs.repo
fi

if [ -f /etc/yum.repos.d/lustre.repo ]; then
  rm -f /etc/yum.repos.d/lustre.repo
fi  

set -e -x

export install_nifi_on_all_nodes="${install_nifi_on_all_nodes:-true}"
export use_default_configs="${use_default_configs:-true}"
export ambari_services=${ambari_services:-ZOOKEEPER NIFI KAFKA STORM LOGSEARCH AMBARI_METRICS AMBARI_INFRA}
export ambari_password=${ambari_password:-admin}
export cluster_name=${cluster_name:-HDF}
export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-1.8.0-openjdk.x86_64/}
export ranger_user="${ranger_user:-rangeradmin}"
export ranger_pass="${ranger_pass:-BadPass#1}"
export host_count=${host_count:-ask}
export setup_kdc="${setup_kdc:-false}"
export setup_postgres_for_ranger="${setup_postgres_for_ranger:-true}"
export host_os=${host_os:-centos6}
export ambari_stack_version=${ambari_stack_version:-2.1}
export ambari_password=${ambari_password:-BadPass#1}     ##  For security purposes, when installing on AWS, this password will be overridden with your AWS accountid
export ambari_version=2.5.1.0                            ## don't use Ambari 2.5.2.0 for HDF, there is a bug
export hdf_ambari_mpack_url="http://public-repo-1.hortonworks.com/HDF/${host_os}/3.x/updates/3.0.1.1/tars/hdf_ambari_mp/hdf-ambari-mpack-3.0.1.1-5.tar.gz"
#export hdf_ambari_mpack_url="http://public-repo-1.hortonworks.com/HDF/${host_os}/2.x/updates/2.1.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-2.1.0.0-165.tar.gz"
#export hdf_ambari_mpack_url="http://public-repo-1.hortonworks.com/HDF/${host_os}/2.x/updates/2.0.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-2.0.0.0-579.tar.gz"
#export ambari_repo="http://public-repo-1.hortonworks.com/ambari/${host_os}/2.x/updates/${ambari_version}/ambari.repo"
#export hdf_repo_url="http://public-repo-1.hortonworks.com/HDF/${host_os}/2.x/updates/2.0.0.0"


#service user for Ambari to start/stop services on boot
export service_user="demokitadmin"
export service_password="BadPass#1"

yum install -y git python-argparse
cd ~
sudo git clone https://github.com/seanorama/ambari-bootstrap.git

export install_ambari_server=true
chmod +x ~/ambari-bootstrap/ambari-bootstrap.sh
~/ambari-bootstrap/ambari-bootstrap.sh

sleep 20

#create demokitadmin user
curl -iv -u admin:admin -H "X-Requested-By: blah" -X POST -d "{\"Users/user_name\":\"${service_user}\",\"Users/password\":\"${service_password}\",\"Users/active\":\"true\",\"Users/admin\":\"true\"}" http://localhost:8080/api/v1/users

#if running on AWS, fetch accountId
if [ -f /sys/hypervisor/uuid ] && [ `head -c 3 /sys/hypervisor/uuid` == ec2 ]; then
  echo "AWS detected, reading accountId..."
  eval $(curl -sSL http://169.254.169.254/latest/dynamic/instance-identity/document \
    | awk -F\" '/:/ {print "export "$2"="$4}')

  #if accountId not empty, use it as password for admin user
  echo "Overriding ambari_password to AWS accountid..."
  if [ -n "${accountId}" ]; then
    export ambari_password=${accountId} 
  fi  

else
    echo "non-AWS detecting. Leaving password to default"
fi

#update admin password
curl -iv -u admin:admin -H "X-Requested-By: blah" -X PUT -d "{ \"Users\": { \"user_name\": \"admin\", \"old_password\": \"admin\", \"password\": \"${ambari_password}\" }}" http://localhost:8080/api/v1/users/admin


ambari-server stop

echo yes | ambari-server install-mpack --mpack=${hdf_ambari_mpack_url} --purge --verbose

#Optional - modify stack advisor to recommend installing Nifi on all nodes
if [ "${install_nifi_on_all_nodes}" = true ]; then
  cp /var/lib/ambari-server/resources/stacks/HDF/2.0/services/stack_advisor.py /var/lib/ambari-server/resources/stacks/HDF/2.0/services/stack_advisor.py.bak
  sed -i.bak  "s#return \['ZOOKEEPER_SERVER', 'METRICS_COLLECTOR'\]#return \['ZOOKEEPER_SERVER', 'METRICS_COLLECTOR', 'NIFI_MASTER'\]#" /var/lib/ambari-server/resources/stacks/HDF/2.0/services/stack_advisor.py
  sed -i.bak  "s#\('ZOOKEEPER_SERVER': {\"min\": 3},\)#\1\n      'NIFI_MASTER': {\"min\": $host_count},#g"  /var/lib/ambari-server/resources/stacks/HDF/2.0/services/stack_advisor.py
fi

#Optional - setup KDC using automation from https://gist.github.com/abajwa-hw/f8b83e1c12abb1564531e00836b098fa

if [ "${setup_kdc}" = true ]; then
  echo "Setting up KDC..."
  curl -sSL https://gist.github.com/abajwa-hw/f8b83e1c12abb1564531e00836b098fa/raw | sudo -E sh
fi

#ranger pre-reqs
if [ "${setup_postgres_for_ranger}" = true ]; then
  echo "Setting up postgres for Ranger..."
  yum install -y postgresql-jdbc*
  chmod 644 /usr/share/java/postgresql-jdbc.jar
  echo "CREATE DATABASE ranger;" | sudo -u postgres psql -U postgres
  echo "CREATE USER ${ranger_user} WITH PASSWORD '${ranger_pass}';" | sudo -u postgres psql -U postgres
  echo "ALTER DATABASE ranger OWNER TO ${ranger_user};" | sudo -u postgres psql -U postgres 
  echo "GRANT ALL PRIVILEGES ON DATABASE ranger TO ${ranger_user};" | sudo -u postgres psql -U postgres 

  sed -i.bak s/ambari,mapred/${ranger_user},ambari,mapred/g /var/lib/pgsql/data/pg_hba.conf
  cat /var/lib/pgsql/data/postgresql.conf | grep listen_addresses
  #make sure listen_addresses='*'
  ambari-server setup --jdbc-db=postgres --jdbc-driver=/usr/share/java/postgresql-jdbc.jar
  service postgresql restart
fi

#start Ambari
ambari-server start
sleep 30

cd ~


#any customizations?
cd ~/ambari-bootstrap/deploy/

#whether to test with default configs or custom
if [ "${use_default_configs}" = true ]; then
  tee configuration-custom.json > /dev/null << EOF
{
  "configurations" : {
    "nifi-ambari-config": {
       "nifi.security.encrypt.configuration.password": "changemeplease"
    }
  }
}
EOF

else

  tee configuration-custom.json > /dev/null << EOF
{
  "configurations" : {
    "nifi-ambari-config": {
        "nifi.content.repository.dir.default": "/nifi/content_repository",
        "nifi.database.dir": "/nifi/database_repository",
        "nifi.flowfile.repository.dir": "/nifi/flowfile_repository",
        "nifi.internal.dir": "/nifi",
        "nifi.provenance.repository.dir.default": "/nifi/provenance_repository",        
        "nifi.max_mem": "1g",        
        "nifi.node.port": "9092",                
        "nifi.node.protocol.port": "9089",                        
        "nifi.node.ssl.port": "9093",
        "nifi.security.encrypt.configuration.password": "changemeplease"
    },
    "nifi-env": {
        "nifi_user": "mynifiuser",
        "nifi_group": "mynifigroup"
    },    
    "nifi-properties": {
        "nifi.security.identity.mapping.pattern.dn": "^CN=(.*?), OU=(.*?)$",
        "nifi.security.identity.mapping.value.dn": "\$1@\$2"
    }
  }
}
EOF
fi

echo "Deploying HDF..."
export ambari_stack_name=HDF
./deploy-recommended-cluster.bash



#To enable security:
#1. enable SSL: https://community.hortonworks.com/articles/58009/hdf-20-enable-ssl-for-apache-nifi-from-ambari.html
#2. enable Ranger: 
#         https://community.hortonworks.com/articles/58769/hdf-20-enable-ranger-authorization-for-hdf-compone.html
#         https://community.hortonworks.com/articles/60001/hdf-20-integrating-secured-nifi-with-secured-range.html
#3. To enable Kerberos:
#         https://community.hortonworks.com/articles/60186/hdf-20-use-ambari-to-enable-kerberos-for-hdf-clust-1.html
#         https://community.hortonworks.com/articles/58793/hdf-20-use-ambari-to-enable-kerberos-for-hdf-clust.html


#To reset and start over:
#python /usr/lib/python2.6/site-packages/ambari_agent/HostCleanup.py -s
#ambari-server stop
#ambari-server reset
#  ##type yes twice
#ambari-agent stop
#yum remove -y ambari-server ambari-agent
#rm -rf /root/*
#rm -rf /var/lib/ambari-server/resources/host_scripts/nifi-certs
#kdb5_util destroy
#  ##type yes