| # One liner to install HDP Erie TP on a single node | |
| # export host_count=1; curl -sSL https://gist.github.com/abajwa-hw/8930ff5eee82285930a4c2f697310729/raw | sudo -E sh | |
| export ambari_services="ZOOKEEPER HDFS YARN MAPREDUCE2 HIVE PIG SLIDER TEZ" | |
| export cluster_name=HDP | |
| export ambari_password="admin" | |
| #TP 6 | |
| export ambari_repo="http://public-repo-1.hortonworks.com/HDP-LABS/Projects/Erie-Preview/ambari/2.4.0.0-7/centos6/ambari.repo" |
| # - Run the below script. You can customize the functionality by modifying the exports below. Otherwise to run the script with defaults, just run below one-liner and wait for 30 min: | |
| # wget https://gist.github.com/abajwa-hw/ae026c63260ed6284b3d0333eac2c22d/raw -O setup_hdf.sh ; chmod +x setup_hdf.sh; export hdf_build=551; ./setup_hdf.sh ; | |
| export ranger_user="${ranger_user:-rangeradmin}" | |
| export ranger_pass="${ranger_pass:-BadPass#1}" | |
| export nifi_cert_pass="${nifi_cert_pass:-BadPass#1}" | |
| export hdf_build="${hdf_build:-579}" | |
| read -p "Git Userid: " git_user | |
| read -s -p "Git Password: " git_password |
| #!/usr/local/bin/python | |
| def post_request(url, body): | |
| import json, urllib2 | |
| encoded_body = json.dumps(body) | |
| req = urllib2.Request(str(url), encoded_body) | |
| req.get_method = lambda: 'PUT' | |
| try: | |
| response = urllib2.urlopen(req, encoded_body).read() | |
| except urllib2.HTTPError, error: |
| <?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<flowController encoding-version=\"1.0\">\n <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount>\n <maxEventDrivenThreadCount>5</maxEventDrivenThreadCount>\n <rootGroup>\n <id>7c84501d-d10c-407c-b9f3-1d80e38fe36a</id>\n <name>NiFi Flow</name>\n <position x=\"0.0\" y=\"0.0\"/>\n <comment/>\n <processGroup>\n <id>f5ca9391-0f9a-4e95-35d7-0fc2188fe61b</id>\n <name>Twitter Dashboard - Push tweets into HDFS/Solr</name>\n <position x=\"469.0\" y=\"166.0\"/>\n <comment/>\n <processor>\n <id>f6327477-fb7d-4af0-bae5-88e29c322093</id>\n <name>PutSolrContentStream</name>\n <position x=\"570.1624606053042\" y=\"398.4867144979376\"/>\n <styles/>\n <comment/>\n <class>org.apache.nifi.processors.solr.PutSolrContentStream</class>\n <maxConcurrentTasks>1</maxConcurrentTasks>\n <schedulingPeriod>0 sec</schedulingPeriod>\n <penalizationPeriod>30 sec</penaliza |
| # Script to install HDP 2.5, Nifi 1.0, Zeppein with Truckin demo and Nifi-Twitter demo. Run below command to kick off: | |
| #curl -sSL https://gist.github.com/abajwa-hw/3f2e211d252bba6cad6a6735f78a4a93/raw | sudo -E sh | |
| #To run on multinode: run below on non-ambariserver nodes first | |
| #export ambari_server=<FQDN of host running ambari-server>; export ambari_version=2.4.1.0; curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh ; | |
| bootstrap_home=/tmp | |
| rm -rf ${bootstrap_home}/ambari-bootstrap | |
| set -e -x | |
| export hdp_ver=${hdp_ver:-2.5} #version of HDP |
| #Script to onboard users for sandbox tutorials. One liner to download and run: | |
| #curl -sSL https://gist.github.com/abajwa-hw/176aa9f7a43d46068206cbdb96179e00/raw | sudo -E bash | |
| export ambari_pass=${ambari_pass:-'IBMDem0s!'} #ambari password | |
| export refresh_notebooks=${refresh_notebooks:-'false'} #whether to pull latest Zeppelin notebooks from github. Disabling as there may not be access | |
| ambari_url="https://localhost:8080/api/v1" | |
| echo "Turning off iptables..." | |
| sudo service iptables stop |
yum install -y gcc gcc-cpp gcc-c++
| #Based on http://docs.hortonworks.com/HDPDocuments/HDF2/HDF-2.0.0/bk_ambari-installation/content/index.html | |
| export host_os=centos7 | |
| export ambari_version=2.4.1.0 | |
| export hdf_ambari_mpack_url="http://public-repo-1.hortonworks.com/HDF/${host_os}/2.x/updates/2.0.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-2.0.0.0-579.tar.gz" | |
| export ambari_repo="http://public-repo-1.hortonworks.com/ambari/${host_os}/2.x/updates/${ambari_version}/ambari.repo" | |
| #1. turn off firewall | |
| systemctl stop firewalld.service | |
| systemctl disable firewalld |
| #install HDP 2.6 | |
| export ambari_services="HDFS HIVE PIG SPARK MAPREDUCE2 TEZ YARN ZOOKEEPER ZEPPELIN KAFKA STORM HBASE PHOENIX" | |
| curl -sSL https://gist.github.com/abajwa-hw/7794ea013c96f3f41c4a8b10aeeccd4d/raw | sudo -E sh | |
| #Setup Mysql | |
| host=$(hostname -f) |
| #To run, first copy DMX ambari package to /tmp | |
| # scp -i ~/.ssh/field.pem ~/Downloads/dmexpress-9.2-el7.ambari-service_en.bin centos@172.26.194.217:/tmp | |
| #then export any variables and then execute below: | |
| # curl -sSL https://gist.github.com/abajwa-hw/a4bec6c5fe7545f3515893437746fd38/raw | sudo -E sh | |
| #For multinode, prep the other hosts first by running below (replace ambari.xxx.com) | |
| # sudo rpm -Uvh http://dev.mysql.com/get/mysql-community-release-el7-5.noarch.rpm | |
| # export ambari_version="2.6.1.0";export ambari_server="ambari.xxx.com";curl -sSL https://raw.githubusercontent.com/seanorama/ambari-bootstrap/master/ambari-bootstrap.sh | sudo -E sh | |
| #set -x |