Skip to content

Instantly share code, notes, and snippets.

View abajwa-hw's full-sized avatar

Ali Bajwa abajwa-hw

219 followers · 15 following
View GitHub Profile
@abajwa-hw
abajwa-hw / deploy_vanilla_hdp.sh
Last active May 10, 2020 07:01
vanilla HDP install
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
#To run - export any variables then execute below:
#export cluster_name=DESIRED_CLUSTER_NAME; curl -sSL https://gist.github.com/abajwa-hw/7794ea013c96f3f41c4a8b10aeeccd4d/raw | sudo -E sh
export cluster_name=${cluster_name:-hdp}
export ambari_password=${ambari_password:-BadPass#1} #Ambari password
export host_count=${host_count:-1} #choose number of nodes
export ambari_services=${ambari_services:-HDFS HIVE PIG MAPREDUCE2 TEZ YARN ZOOKEEPER ZEPPELIN AMBARI_INFRA_SOLR KAFKA SPARK2} #AMBARI_METRICS can be added post-install
export hdp_ver=${hdp_ver:-3.1}
export ambari_version=2.7.3.0
@abajwa-hw
abajwa-hw / settings.xml
Last active February 7, 2020 13:07
~/.m2/settings.xml
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
http://maven.apache.org/xsd/settings-1.0.0.xsd">
<localRepository/>
<interactiveMode/>
<usePluginRegistry/>
<offline/>
<pluginGroups/>
<servers/>
@abajwa-hw
abajwa-hw / setup_kdc.sh
Last active October 5, 2019 17:43
Setup KDC on Ambari node
# curl -sSL https://gist.github.com/abajwa-hw/f8b83e1c12abb1564531e00836b098fa/raw | sudo -E sh
export host=$(hostname -f)
export realm=${realm:-HORTONWORKS.COM}
export domain=${domain:-hortonworks.com}
export kdcpassword=${kdcpassword:-BadPass#1}
set -e
sudo yum -y install krb5-server krb5-libs krb5-auth-dialog krb5-workstation
@abajwa-hw
abajwa-hw / deploy_hdp31hdf33.sh
Last active July 16, 2019 18:26
Deploy HDP 3.1 and HDF 3.3
#!/usr/bin/env bash
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/d7cd1c0232c1af46ee2c465e4871ddc6/raw | sudo -E sh
export create_image=${create_image:-false}
export ambari_version=2.7.3.0
export mpack_url="http://public-repo-1.hortonworks.com/HDF/amazonlinux2/3.x/updates/3.3.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.3.0.0-165.tar.gz"
export hdf_vdf="http://s3.amazonaws.com/public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.3.0.0/HDF-3.3.0.0-165.xml"
export minifi_java="http://public-repo-1.hortonworks.com/HDF/3.3.0.0/minifi-0.6.0.3.3.0.0-165-bin.tar.gz"
@abajwa-hw
abajwa-hw / setup_trucking_demo.sh
Last active May 2, 2019 01:37
Deploy trucking demo on single node HDP 2.5 (Centos 6)
#assuming HDP 2.5 single node already installed with Hbase/Storm/Kafka started
export demo_root=~
export JAVA_HOME=/usr/java/default
cd ${demo_root}
sudo yum -y groupinstall "Development Tools"
sudo yum install -y wget git
@abajwa-hw
abajwa-hw / deploy_hdf33.sh
Last active April 13, 2019 00:37
Automation to deploy HDF 3.3 on RHEL 7
#!/usr/bin/env bash
# Launch Centos/RHEL 7 Vm with at least 4 cores / 16Gb mem / 60Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/b5565d7e7f9beffd8dd57a970dc54266/raw | sudo -E sh
export ambari_password=${ambari_password:-StrongPassword}
export db_password=${db_password:-StrongPassword}
export nifi_password=${nifi_password:-StrongPassword}
export ambari_services="ZOOKEEPER STREAMLINE NIFI KAFKA STORM REGISTRY NIFI_REGISTRY KNOX AMBARI_METRICS"
export cluster_name=${cluster_name:-hdf}
@abajwa-hw
abajwa-hw / start_hdp_on_boot.sh
Last active December 21, 2018 02:19
Script to setup startup of HDP on boot
#curl -sSL https://gist.github.com/abajwa-hw/4bf004d0fb065d404760eaeebc15e74e/raw | sudo -E sh
#################################
## write out start_services.sh #
#################################
cat > /root/start_services.sh <<-'EOS'
export ambari_user=demokitadmin
export ambari_pass=BadPass#1
@abajwa-hw
abajwa-hw / flow.xml
Last active December 14, 2018 06:53
flow.xml for twitter including atlas reporting task
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<flowController encoding-version="1.3">
<maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>
<maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>
<registries/>
<rootGroup>
<id>94d19823-0163-1000-97c6-e7c5579e4a0c</id>
<name>NiFi Flow</name>
<position x="0.0" y="0.0"/>
<comment/>
@abajwa-hw
abajwa-hw / deploy_hdp_nifi.sh
Last active December 12, 2018 00:50
Deploy vanilla HDP plus Nifi on single node
#!/usr/bin/env bash
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
# Then run:
# export cluster_name=DESIRED_CLUSTER_NAME ; curl -sSL https://gist.github.com/abajwa-hw/bbe2bdd1ed6a0f738a90dd4e07480e3b/raw | sudo -E sh
export cluster_name=${cluster_name:-hdp}
export ambari_password=${ambari_password:-BadPass#1} #Ambari password
export host_count=${host_count:-1} #choose number of nodes
export ambari_services=${ambari_services:-HDFS HIVE PIG SPARK MAPREDUCE2 TEZ YARN ZOOKEEPER ZEPPELIN NIFI AMBARI_INFRA_SOLR KAFKA SPARK2} #AMBARI_METRICS can be added post-install
export hdp_ver=${hdp_ver:-3.1}
@abajwa-hw
abajwa-hw / deploy_hdp30hdf33.sh
Last active December 5, 2018 15:44
Deploy HDP 3.0 and HDF 3.3
#!/usr/bin/env bash
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/66c62bc860a47dfb0de53dfe5cbb4415/raw | sudo -E sh
export create_image=${create_image:-false}
export ambari_version=2.7.1.0
export mpack_url="http://public-repo-1.hortonworks.com/HDF/amazonlinux2/3.x/updates/3.3.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.3.0.0-165.tar.gz"
export hdf_vdf="http://s3.amazonaws.com/public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.3.0.0/HDF-3.3.0.0-165.xml"