Skip to content

Instantly share code, notes, and snippets.

Last active November 2, 2018 18:00
Show Gist options
  • Save abajwa-hw/ae4125c5154deac6713cdd25d2b83620 to your computer and use it in GitHub Desktop.
Save abajwa-hw/ae4125c5154deac6713cdd25d2b83620 to your computer and use it in GitHub Desktop.
Setup HDF via blueprints
# Sample script to deploy HDF via blueprint
# - Installs Ambari server/agents
# - Installs HDF mpack
# - Uses ambari-bootstrap to generate blueprint based on stack advisor recommendation and starts cluster install
# - Optionally: installs KDC, sets up postgres for Ranger, allows customizations of config properties and number of Nifi nodes
# Usage: su as root and run below to invoke this script on a host where CentOS/RHEL has been freshly installed (do NOT run this on HDP sandbox!). You can customize the functionality by setting env vars e.g.
# export host_count=3; export install_nifi_on_all_nodes=true; curl -sSL | sudo -E sh ;
# Note for multi-node install, you will need to install/register agents on other nodes first using below (replace <AMBARI_SERVER_HOSTNAME>)
# export ambari_server=<AMBARI_SERVER_HOSTNAME>; curl -sSL | sudo -E sh ;
# e.g.
# export ambari_server=abajwa-hdf-qe-bp-1.openstacklocal; export ambari_version=; curl -sSL | sudo -E sh ;
# see for more details:
#clean previous setup
rm -rf ~/ambari-bootstrap
rm -f *_payload
rm -rf ~/hdf_ambari_mp
rm ~/apache-maven-3.0.5-bin.tar.gz
#remove unneeded repos
if [ -f /etc/yum.repos.d/zfs.repo ]; then
rm -f /etc/yum.repos.d/zfs.repo
if [ -f /etc/yum.repos.d/lustre.repo ]; then
rm -f /etc/yum.repos.d/lustre.repo
set -e -x
export install_nifi_on_all_nodes="${install_nifi_on_all_nodes:-true}"
export use_default_configs="${use_default_configs:-true}"
export ambari_password=${ambari_password:-admin}
export cluster_name=${cluster_name:-HDF}
export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-1.8.0-openjdk.x86_64/}
export ranger_user="${ranger_user:-rangeradmin}"
export ranger_pass="${ranger_pass:-BadPass#1}"
export host_count=${host_count:-ask}
export setup_kdc="${setup_kdc:-false}"
export setup_postgres_for_ranger="${setup_postgres_for_ranger:-true}"
export host_os=${host_os:-centos6}
export ambari_stack_version=${ambari_stack_version:-2.1}
export ambari_password=${ambari_password:-BadPass#1} ## For security purposes, when installing on AWS, this password will be overridden with your AWS accountid
export ambari_version= ## don't use Ambari for HDF, there is a bug
export hdf_ambari_mpack_url="${host_os}/3.x/updates/"
#export hdf_ambari_mpack_url="${host_os}/2.x/updates/"
#export hdf_ambari_mpack_url="${host_os}/2.x/updates/"
#export ambari_repo="${host_os}/2.x/updates/${ambari_version}/ambari.repo"
#export hdf_repo_url="${host_os}/2.x/updates/"
#service user for Ambari to start/stop services on boot
export service_user="demokitadmin"
export service_password="BadPass#1"
yum install -y git python-argparse
cd ~
sudo git clone
export install_ambari_server=true
chmod +x ~/ambari-bootstrap/
sleep 20
#create demokitadmin user
curl -iv -u admin:admin -H "X-Requested-By: blah" -X POST -d "{\"Users/user_name\":\"${service_user}\",\"Users/password\":\"${service_password}\",\"Users/active\":\"true\",\"Users/admin\":\"true\"}" http://localhost:8080/api/v1/users
#if running on AWS, fetch accountId
if [ -f /sys/hypervisor/uuid ] && [ `head -c 3 /sys/hypervisor/uuid` == ec2 ]; then
echo "AWS detected, reading accountId..."
eval $(curl -sSL \
| awk -F\" '/:/ {print "export "$2"="$4}')
#if accountId not empty, use it as password for admin user
echo "Overriding ambari_password to AWS accountid..."
if [ -n "${accountId}" ]; then
export ambari_password=${accountId}
echo "non-AWS detecting. Leaving password to default"
#update admin password
curl -iv -u admin:admin -H "X-Requested-By: blah" -X PUT -d "{ \"Users\": { \"user_name\": \"admin\", \"old_password\": \"admin\", \"password\": \"${ambari_password}\" }}" http://localhost:8080/api/v1/users/admin
ambari-server stop
echo yes | ambari-server install-mpack --mpack=${hdf_ambari_mpack_url} --purge --verbose
#Optional - modify stack advisor to recommend installing Nifi on all nodes
if [ "${install_nifi_on_all_nodes}" = true ]; then
cp /var/lib/ambari-server/resources/stacks/HDF/2.0/services/ /var/lib/ambari-server/resources/stacks/HDF/2.0/services/
sed -i.bak "s#return \['ZOOKEEPER_SERVER', 'METRICS_COLLECTOR'\]#return \['ZOOKEEPER_SERVER', 'METRICS_COLLECTOR', 'NIFI_MASTER'\]#" /var/lib/ambari-server/resources/stacks/HDF/2.0/services/
sed -i.bak "s#\('ZOOKEEPER_SERVER': {\"min\": 3},\)#\1\n 'NIFI_MASTER': {\"min\": $host_count},#g" /var/lib/ambari-server/resources/stacks/HDF/2.0/services/
#Optional - setup KDC using automation from
if [ "${setup_kdc}" = true ]; then
echo "Setting up KDC..."
curl -sSL | sudo -E sh
#ranger pre-reqs
if [ "${setup_postgres_for_ranger}" = true ]; then
echo "Setting up postgres for Ranger..."
yum install -y postgresql-jdbc*
chmod 644 /usr/share/java/postgresql-jdbc.jar
echo "CREATE DATABASE ranger;" | sudo -u postgres psql -U postgres
echo "CREATE USER ${ranger_user} WITH PASSWORD '${ranger_pass}';" | sudo -u postgres psql -U postgres
echo "ALTER DATABASE ranger OWNER TO ${ranger_user};" | sudo -u postgres psql -U postgres
echo "GRANT ALL PRIVILEGES ON DATABASE ranger TO ${ranger_user};" | sudo -u postgres psql -U postgres
sed -i.bak s/ambari,mapred/${ranger_user},ambari,mapred/g /var/lib/pgsql/data/pg_hba.conf
cat /var/lib/pgsql/data/postgresql.conf | grep listen_addresses
#make sure listen_addresses='*'
ambari-server setup --jdbc-db=postgres --jdbc-driver=/usr/share/java/postgresql-jdbc.jar
service postgresql restart
#start Ambari
ambari-server start
sleep 30
cd ~
#any customizations?
cd ~/ambari-bootstrap/deploy/
#whether to test with default configs or custom
if [ "${use_default_configs}" = true ]; then
tee configuration-custom.json > /dev/null << EOF
"configurations" : {
"nifi-ambari-config": {
"": "changemeplease"
tee configuration-custom.json > /dev/null << EOF
"configurations" : {
"nifi-ambari-config": {
"nifi.content.repository.dir.default": "/nifi/content_repository",
"nifi.database.dir": "/nifi/database_repository",
"nifi.flowfile.repository.dir": "/nifi/flowfile_repository",
"nifi.internal.dir": "/nifi",
"nifi.provenance.repository.dir.default": "/nifi/provenance_repository",
"nifi.max_mem": "1g",
"nifi.node.port": "9092",
"nifi.node.protocol.port": "9089",
"nifi.node.ssl.port": "9093",
"": "changemeplease"
"nifi-env": {
"nifi_user": "mynifiuser",
"nifi_group": "mynifigroup"
"nifi-properties": {
"": "^CN=(.*?), OU=(.*?)$",
"": "\$1@\$2"
echo "Deploying HDF..."
export ambari_stack_name=HDF
#To enable security:
#1. enable SSL:
#2. enable Ranger:
#3. To enable Kerberos:
#To reset and start over:
#python /usr/lib/python2.6/site-packages/ambari_agent/ -s
#ambari-server stop
#ambari-server reset
# ##type yes twice
#ambari-agent stop
#yum remove -y ambari-server ambari-agent
#rm -rf /root/*
#rm -rf /var/lib/ambari-server/resources/host_scripts/nifi-certs
#kdb5_util destroy
# ##type yes
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment