Skip to content

Instantly share code, notes, and snippets.

View abajwa-hw's full-sized avatar

Ali Bajwa abajwa-hw

219 followers · 15 following
View GitHub Profile
@abajwa-hw
abajwa-hw / setup_kdc.sh
Last active September 13, 2020 11:40
Setup KDC
# curl -sSL https://gist.github.com/abajwa-hw/bca3d23fe146c3ebd59a9b5fd19480a3/raw | sudo -E sh
export host=$(hostname -f)
export realm=${realm:-CLOUDERA.COM}
export domain=${domain:-cloudera.com}
export kdcpassword=${kdcpassword:-BadPass#1}
set -e
sudo yum -y install krb5-server krb5-libs krb5-auth-dialog krb5-workstation
@abajwa-hw
abajwa-hw / deploy_hdp31hdf34.sh
Last active October 29, 2020 21:31
Deploy HDP 3.1 and HDF 3.4
#!/usr/bin/env bash
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/9bf11bcaadaa7f94f9075432ce5b3d0b/raw | sudo -E sh
export create_image=${create_image:-true}
export ambari_version=2.7.3.0
#export mpack_url="http://public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.4.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.4.0.0-155.tar.gz"
#export mpack_url="http://public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.4.1.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.4.1.0-5.tar.gz"
export mpack_url="http://public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.4.1.1/tars/hdf_ambari_mp/hdf-ambari-mpack-3.4.1.1-4.tar.gz"
@abajwa-hw
abajwa-hw / deploy_hdf34.sh
Last active October 29, 2020 21:29
Automation to deploy HDF 3.4 on RHEL 7
#!/usr/bin/env bash
# Launch Centos/RHEL 7 Vm with at least 4 cores / 16Gb mem / 60Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/c37d0e847054cf519813066401c33388/raw | sudo -E sh
export ambari_password=${ambari_password:-StrongPassword}
export db_password=${db_password:-StrongPassword}
export nifi_password=${nifi_password:-StrongPassword}
export ambari_services="ZOOKEEPER STREAMLINE NIFI KAFKA STORM REGISTRY NIFI_REGISTRY KNOX AMBARI_METRICS"
export cluster_name=${cluster_name:-hdf}
@abajwa-hw
abajwa-hw / deploy_hdp31hdf33.sh
Last active July 16, 2019 18:26
Deploy HDP 3.1 and HDF 3.3
#!/usr/bin/env bash
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/d7cd1c0232c1af46ee2c465e4871ddc6/raw | sudo -E sh
export create_image=${create_image:-false}
export ambari_version=2.7.3.0
export mpack_url="http://public-repo-1.hortonworks.com/HDF/amazonlinux2/3.x/updates/3.3.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.3.0.0-165.tar.gz"
export hdf_vdf="http://s3.amazonaws.com/public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.3.0.0/HDF-3.3.0.0-165.xml"
export minifi_java="http://public-repo-1.hortonworks.com/HDF/3.3.0.0/minifi-0.6.0.3.3.0.0-165-bin.tar.gz"
@abajwa-hw
abajwa-hw / deploy_hdf33.sh
Last active April 13, 2019 00:37
Automation to deploy HDF 3.3 on RHEL 7
#!/usr/bin/env bash
# Launch Centos/RHEL 7 Vm with at least 4 cores / 16Gb mem / 60Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/b5565d7e7f9beffd8dd57a970dc54266/raw | sudo -E sh
export ambari_password=${ambari_password:-StrongPassword}
export db_password=${db_password:-StrongPassword}
export nifi_password=${nifi_password:-StrongPassword}
export ambari_services="ZOOKEEPER STREAMLINE NIFI KAFKA STORM REGISTRY NIFI_REGISTRY KNOX AMBARI_METRICS"
export cluster_name=${cluster_name:-hdf}
@abajwa-hw
abajwa-hw / deploy_hdp30hdf33.sh
Last active December 5, 2018 15:44
Deploy HDP 3.0 and HDF 3.3
#!/usr/bin/env bash
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/66c62bc860a47dfb0de53dfe5cbb4415/raw | sudo -E sh
export create_image=${create_image:-false}
export ambari_version=2.7.1.0
export mpack_url="http://public-repo-1.hortonworks.com/HDF/amazonlinux2/3.x/updates/3.3.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.3.0.0-165.tar.gz"
export hdf_vdf="http://s3.amazonaws.com/public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.3.0.0/HDF-3.3.0.0-165.xml"
@abajwa-hw
abajwa-hw / setup_knoxSSO_hdp265.sh
Created September 24, 2018 22:43
Setup Knox SSO for Ambari/Ranger/Atlas on HDP 2.6.5
#!/usr/bin/env bash
export cluster_name=$1
export host=$(hostname -f)
export ambari_pass=$2
export knox_ldap_pass=$3
hostname=$(hostname -f)
current_dir=$(pwd)
cd /tmp
#Copy config-update to /tmp
cp /tmp/masterclass/ranger-atlas/HortoniaMunichSetup/config_update.py .
@abajwa-hw
abajwa-hw / deploy_hdp30hdf32.sh
Last active October 25, 2018 02:01
Deploy HDP 3.0 and HDF 3.2
#!/usr/bin/env bash
# Launch Centos/RHEL 7 VM with at least 8 vcpu / 32Gb+ memory / 100Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/2050a71363f14a59bc19c6697cb78a1f/raw | sudo -E sh
export create_image=${create_image:-false}
export ambari_version=2.7.1.0
export mpack_url="http://public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.2.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.2.0.0-520.tar.gz"
export hdf_vdf="http://s3.amazonaws.com/public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.2.0.0/HDF-3.2.0.0-520.xml"
@abajwa-hw
abajwa-hw / configuration-custom.json
Created September 7, 2018 23:15
configuration-custom.json for Ambari 2.7/HDP 3.0/HDF 3.2
{
"configurations": {
"core-site": {
"fs.trash.interval": "4320"
},
"ams-grafana-env": {
"metrics_grafana_password": "StrongPassword"
},
"hdfs-site": {
"dfs.replication" : "1",
@abajwa-hw
abajwa-hw / deploy_hdf32.sh
Last active September 8, 2018 17:01
Automation to deploy HDF 3.2 on RHEL 7
#!/usr/bin/env bash
# Launch Centos/RHEL 7 Vm with at least 4 cores / 16Gb mem / 60Gb disk
# Then run:
# curl -sSL https://gist.github.com/abajwa-hw/231bd92e790bac680c40a3dcd7c5a2c6/raw | sudo -E sh
export ambari_password=${ambari_password:-StrongPassword}
export db_password=${db_password:-StrongPassword}
export nifi_password=${nifi_password:-StrongPassword}
export ambari_services="ZOOKEEPER STREAMLINE NIFI KAFKA STORM REGISTRY NIFI_REGISTRY KNOX AMBARI_METRICS"
export cluster_name=${cluster_name:-hdf}