CM Path B Installation

0_prechecks.sh

#!/bin/sh

source ./control.sh

function no_hugepages() {
  echo never > /sys/kernel/mm/transparent_hugepage/enabled
  echo never > /sys/kernel/mm/transparent_hugepage/defrag
}

function verify_capacity() {
  :
}

function verify_ntpd_on() {
  :
}

function verify_nscd_on() {
  :
}

function verify_dns_resolves() {
  :
}

function verify_reverse_dns_resolves() {
  :
}

function verify_iptables_off() {
  :
}

1_get_repos.sh

#!/bin/sh

source ./control.sh

function getMySQLRepo() {
  cd /tmp
  wget --quiet —-no-check-certificate https://dev.mysql.com/get/mysql-community-release-el6-5.noarch.rpm
  yum -y localinstall mysql-community-release-el6-5.noarch.rpm
  cd -
}

function getClouderaManagerRepo() {
  cd /etc/yum.repos.d
  wget --quiet http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/cloudera-manager.repo
  cd -
}

function getRepos() {
  say "Installing MySQL yum repo from dev.msql.com" 
  getMySQLRepo
  say "Installing latest Cloudera Manager repo"
  getClouderaManagerRepo
}

getRepos

2_test_repos.sh

#!/bin/sh

source ./control.sh

function cleanYUM() {
  yum clean all
  rm -Rf /var/cache/yum/x86_64
  yum makecache
}

function verifyRepo() {
  yum repolist enabled | grep Cloudera
  yum repolist enabled | grep MySQL
}

function prepRepos() {
  say "Refreshing YUM metadata"
  cleanYUM
  say "Verifying utility repos"
  verifyRepo
}

prepRepos

3_install_pkgs.sh

#!/bin/sh

source ./control.sh

function install() {
  yum info $1
  yum -y install $1
}

function addJDK() {
  install oracle-j2sdk1.7.x86_64
}

function addCMServer() {
  install cloudera-manager-server
}

function addMySQL() {
  install mysql-community-server
}

function installPkgs() {
  say "Installing JDK 1.7..."
  addJDK
  say "Installing Cloudera Manager server"
  addCMServer
  say "Installing MySQL server"
  addMySQL
}

installPkgs

4_install_JDBC.sh

#!/bin/sh

source ./control.sh

function getConnector() {
  cd /tmp
  wget --quiet http://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.35.tar.gz
}

function extractJAR() {
  gunzip /tmp/mysql-connector-java-5.1.35.tar.gz
  tar xvf /tmp/mysql-connector-java-5.1.35.tar
}

function placeJAR() {
  cd /tmp/mysql-connector-java-5.1.35/
  mkdir -p /usr/share/java
  cp mysql-connector-java-5.1.35-bin.jar /usr/share/java
  cd /usr/share/java
  ln mysql-connector-java-5.1.35-bin.jar mysql-connector-java.jar
}

function integrateConnector() {
  # Assumes Cloudera Manager server package is installed
  cm_config=/etc/default/cloudera-scm-server
  jar_name=`grep CMF_JDBC_DRIVER_JAR ${cm_config}`
  echo "${cm_config}: ${jar_name}"
}

function installConnector() {
  say "Installing MySQL JDBC connector..."
  getConnector
  extractJAR
  placeJAR
  say "Configuring CM Server with MySQL JDBC..."
  integrateConnector
}

installConnector

5_configure_MySQL.sh

#!/bin/sh

source ./control.sh

function startMySQL() {
  service mysqld start
}

function secureMySQL() {
  mysql_secure_installation
}

function noIPV6User() {
mysql -u root -p <<EOC
DELETE FROM mysql.user WHERE host='::1';
FLUSH PRIVILEGES;
EOC
}

function createDB() {
  db=$1
  user=$2
  node=$3
  pass=$4

  echo "Creating database ${db}"
  echo "Granting access to ${user} on ${node}"

mysql -u root -p <<EOC
CREATE DATABASE ${db};
GRANT ALL ON ${db}.* TO "${user}"@"${node}" IDENTIFIED BY "${pass}";
EOC
}

function createDBs() {
  admin=`hostname -f`
  edge=${admin}
  createDB scm scm ${admin} cloudera
  createDB rman rman ${admin} cloudera
  createDB hive hive ${admin} cloudera
  createDB oozie oozie ${edge} cloudera
  createDB hue hue ${edge} cloudera
  createDB sentry sentry ${admin} cloudera
}

function integrateDB() {
  path=/usr/share/cmf/schema
  ${path}/scm_prepare_database.sh mysql -h $(hostname -f) --scm-host $(hostname -f) scm scm
}

function configureMySQL() {
  startMySQL
  say "Securing your MySQL server..."
  secureMySQL
  noIPV6User
  say "Creating databases for CM and CDH services..."
  createDBs
  say "Verifying and writing db connection string for CM..."
  integrateDB
}

configureMySQL

6_start_cm.sh

#!/bin/sh

source ./control.sh 

function initCM() {
  service cloudera-scm-server start
}

function testAPI() {
  AUTH="admin:admin"
  # JSON="Content-type: application/json"
  API_URL="http://$(hostname -f):7180/api"
  say "CM API URL Base: ${API_URL}"

  VER=`curl -u ${AUTH} "${API_URL}/version" --silent`
  say "Latest API version is ${VER}"
  echo

  greeting="Greetings!" 
  say "Testing API echo..."
  curl -X GET -u ${AUTH} --silent -i "${API_URL}/${VER}/tools/echo?message=$greeting"
}

function verifyCM() {
  say "Initializing Cloudera Manager service..."
  initCM
  say "Wait until the Jetty service is started to browse"
  say "Use tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log | grep 'Started Jetty server'" 
  say "Sleeping for 45 seconds before testing REST API"
  sleep 45
  say "Verifying CM API version..."
  testAPI
}

verifyCM

control.sh

r='\e[31m'
g='\e[32m'
n='\e[0m'

function say() {
  echo -e "${g}$1${n}"
}

# say "${r}This is red, ${g}this is green, and ${n}this is normal"

gistfile1.txt

function add_parcels() {
# for Anaconda: https://repo.continuum.io/pkgs/misc/parcels/
# for Spark2 CSD: http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.0.0.cloudera2.jar (no .sha file)
# for Spark2 parcel: http://archive.cloudera.com/spark2/parcels/2.0.0.cloudera2/

}

function add_host_template() {
}

function consolidate_role_groups() {
}

Turns out you can't create pull requests on a gist! I've created a fork and applied some small fixes for the MySQL installation, so if you're interested you can merge from there. Cheers

https://gist.github.com/a8ed9b6d24179cf6e9c3769155885c12.git

@MrMaynard turns out a gist tracks all its forks, and the list of forks shows which have been modified.

• GRANT statements now work
• control.sh added, includes function to print updates in green