Setup HDF

setup_hdf.sh

#  - Run the below script. You can customize the functionality by modifying the exports below. Otherwise to run the script with defaults, just run below one-liner and wait for 30 min:
# wget https://gist.github.com/abajwa-hw/ae026c63260ed6284b3d0333eac2c22d/raw -O setup_hdf.sh ; chmod +x setup_hdf.sh; export hdf_build=551; ./setup_hdf.sh ;

export ranger_user="${ranger_user:-rangeradmin}"
export ranger_pass="${ranger_pass:-BadPass#1}"
export nifi_cert_pass="${nifi_cert_pass:-BadPass#1}"
export hdf_build="${hdf_build:-579}"

read -p "Git Userid: " git_user
read -s -p "Git Password: " git_password
#read -p "Comma seperated list of FQDN of hostnames: " HOSTNAMES

#Setup Ambari
chkconfig iptables off
/etc/init.d/iptables stop
#wget http://dev.hortonworks.com.s3.amazonaws.com/ambari/centos6/2.x/updates/2.4.0.1/ambariqe.repo -O /etc/yum.repos.d/ambari.repo
wget http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos6/2.x/latest/2.4.1.0/ambaribn.repo -O /etc/yum.repos.d/ambari.repo
yum clean all

yum install ambari-server -y
ambari-server setup -s


#Optional: ranger pre-reqs
yum install -y postgresql-jdbc*
chmod 644 /usr/share/java/postgresql-jdbc.jar
echo "CREATE DATABASE ranger;" | sudo -u postgres psql -U postgres
echo "CREATE USER ${ranger_user} WITH PASSWORD '${ranger_pass}';" | sudo -u postgres psql -U postgres
echo "ALTER DATABASE ranger OWNER TO ${ranger_user};" | sudo -u postgres psql -U postgres 
echo "GRANT ALL PRIVILEGES ON DATABASE ranger TO ${ranger_user};" | sudo -u postgres psql -U postgres 
#add rangeradmin to pg_hba.conf
sed -i.bak s/ambari,mapred/${ranger_user},ambari,mapred/g /var/lib/pgsql/data/pg_hba.conf
cat /var/lib/pgsql/data/postgresql.conf | grep listen_addresses
#make sure listen_addresses='*'
ambari-server setup --jdbc-db=postgres --jdbc-driver=/usr/share/java/postgresql-jdbc.jar
service postgresql restart

#Build latest mpack

#Install Maven 3.0.5
wget http://mirrors.gigenet.com/apache/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz
su -c "tar -zxvf apache-maven-3.0.5-bin.tar.gz -C /opt/" 
export M2_HOME=/opt/apache-maven-3.0.5
export M2=$M2_HOME/bin
PATH=$M2:$PATH
echo "export M2_HOME=/opt/apache-maven-3.0.5" >> ~/.bashrc
echo "export M2=$M2_HOME/bin" >> ~/.bashrc
echo "PATH=$M2:$PATH" >> ~/.bashrc

#point maven to internal repo
mkdir -p ~/.m2
tee ~/.m2/settings.xml > /dev/null << EOF
<settings>
 <profiles>
   <profile>
     <id>hwxInternal</id>
     <repositories>
       <repository>
         <id>HwxInternal</id>
         <name>HwxInternal</name>
         <url>http://nexus-private.hortonworks.com/nexus/content/groups/public</url>
       </repository>
     </repositories>
   </profile>
 </profiles>
 <activeProfiles>
   <activeProfile>hwxInternal</activeProfile>
 </activeProfiles>
</settings>
EOF

#Clone git
yum install -y git
git clone https://$git_user:$git_password@github.com/hortonworks/hdf_ambari_mp.git


#Build mpack
#export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk.x86_64
export JAVA_HOME=$(find /usr/jdk64 -iname 'jdk1.8*' -type d)
cd hdf_ambari_mp/hdf-ambari-mpack
mvn versions:set -DnewVersion=0.1.0.0-1
mvn clean package -DminAmbariVersion=2.4.0.1 -DmaxAmbariVersion= -Dnifiversion=1.0.0.2.0.0.0-$hdf_build

#this will build the below tarball
ls -la target/hdf-ambari-mpack-0.1.0.0-1.tar.gz


#Install mpack
ambari-server install-mpack --mpack=target/hdf-ambari-mpack-0.1.0.0-1.tar.gz --purge --verbose

#Optional - generate and distribute certs to be used later
#cd ~
#wget https://hipchat.hortonworks.com/files/1/2055/bT1LbKB8SS26X9t/nifi-toolkit-1.0.0-SNAPSHOT-bin.zip
#unzip nifi-toolkit-1.0.0-SNAPSHOT-bin.zip
#mkdir /var/lib/ambari-server/resources/host_scripts/nifi-certs
#/root/nifi-toolkit-1.0.0-SNAPSHOT/bin/certs.sh -o /var/lib/ambari-server/resources/host_scripts/nifi-certs -K ${nifi_cert_pass} -S ${nifi_cert_pass} -T ${nifi_cert_pass} -n $HOSTNAMES

#Start Ambari
ambari-server start

#Optional - setup KDC using automation from https://gist.github.com/abajwa-hw/f8b83e1c12abb1564531e00836b098fa
curl -sSL https://gist.github.com/abajwa-hw/f8b83e1c12abb1564531e00836b098fa/raw | sudo -E sh

# optional - export p.12 certificate to import into your browser before accessing SSL enabled Nifi
#ambari_fqdn=$(hostname -f)
#ambari_hostname=$(hostname)
#today=$(date +"%m-%d")
#keytool -importkeystore -srckeystore /var/lib/ambari-server/resources/host_scripts/nifi-certs/$ambari_fqdn/$ambari_fqdn.jks -srcstoretype JKS -deststoretype PKCS12 -destkeystore $ambari_hostname-$today.p12


echo "Setup complete. Now launch Ambari and start install via wizard.Under repos paste latest HDF repo from http://release.eng.hortonworks.com/hwre-api/latestcompiledbuild?stack=HDF&release=2.0.0.0&platform=linux&os=centos6"
echo "e.g http://public-repo-1.hortonworks.com/HDF/centos6/2.x/updates/2.0.0.0"

# 1. Install HDF
# 2. Enable SSL (see screenshot for values)
#    keystore path: /var/lib/ambari-agent/cache/host_scripts/nifi-certs/{nifi_node_ssl_host}/{nifi_node_ssl_host}.jks
#    truststore path: /var/lib/ambari-agent/cache/host_scripts/nifi-certs/{nifi_node_ssl_host}/truststore.jks
# 3. Install Ranger
# 4. Enable Nifi Ranger plugin
# 5. Check the Nifi ranger repo got correctly created. If not, update and test. Should get 403 (see screenshot)
# 6. SCP .p12 file (generated under /root on ambari node) to your laptop and import generated .p12 into your browser (may want to remove old certs from keychain)
# 7. Restart Chrome and try to open Nifi UI. Should get "Access denied" due to unsufficient permissions and requesting user should show up on Ranger audit
# 8. Create local user in Ranger for requesting user (e.g. OU=apache.nifi, CN=abajwa-hdf-dev-ssl-1.openstacklocal)
# 9. Create policy for user (e.g. OU=apache.nifi, CN=abajwa-hdf-dev-ssl-1.openstacklocal)
#    /flow - read
#    /proxy - read/write