ibspoof/dse_5_0_x_installer.sh

## dse_5_0_x_installer.sh
#!/bin/bash

########################################################################
###                                                                  ###
###       Install Script to install DSE 5.0 on Ubuntu or CentOS      ###
###          created by Brad Vernon brad.vernon@datastax.com         ###
###                                                                  ###
########################################################################

# AWS Instance gathering, do not delete
getAwsMetaData() {
    METADATA_URL_BASE="http://169.254.169.254/latest/meta-data"
    PRIVATE_IP=$(curl --silent ${METADATA_URL_BASE}/local-ipv4)
    PUBLIC_IP=$(curl --silent ${METADATA_URL_BASE}/public-ipv4)
    DC_AND_AZ=$(curl --silent ${METADATA_URL_BASE}/placement/availability-zone)
    DC_AND_AZ_SPLIT=(${DC_AND_AZ//-/ })
    AZ=${DC_AND_AZ_SPLIT[${#DC_AND_AZ_SPLIT[@]} - 1]}
    REGION_NUM=${AZ:0:1}
    DC_NAME=$(echo ${DC_AND_AZ} | rev | cut -c 2- | rev)
    RACK=$(echo ${DC_AND_AZ} | rev | cut -c -2 | rev)
}

getAzureDc() {
    IP=$1
    if [ IP == "10.1."* ]; then
        DC=us-west
    else
        DC=us-east
    fi
}

#Azure install info
getAzureMetaData() {
    METADATA_URL="http://169.254.169.254/metadata/v1/InstanceInfo"
    PRIVATE_IP=$(ifconfig eth0 | grep inet | awk '{print $2}' | sed 's/addr://' | grep .)
    PUBLIC_IP=$(curl --silent ipinfo.io/ip)
    NODE_INFO=$(curl --silent ${METADATA_URL_BASE})
    RACK=$(echo $NODE_INFO | awk -F":" '{print $4}' | awk -F"}" '{print $1}')
}

### Start config

CLOUD=azure # options: azure, aws
OS_TYPE=centos  # options: centos, ubuntu (Note: RH = centos)
UPDATE_SYS_PACKAGES=true
DEFAULT_APT_PKGS="vim sysstat wget htop python-pip python-dev libev4 libev-dev ntp"

# Username and pass for academy to download installer
DSE_ACADEMY_USER=
DSE_ACADEMY_PASS=
# DSE and Opscenter Agent Versions
DSE_VERSION=5.0.5
OPSC_VERSION=6.0.7

## Latest version visit http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
RPM_JAVA_LINK=http://download.oracle.com/otn-pub/java/jdk/8u112-b15/jdk-8u112-linux-x64.rpm


## drives
## RAIDed drives can be EBS or local SSDs
declare -A raid
raid[1][name]=MY_RAID
raid[1][target]=/dev/md0
raid[1][devices]="/dev/xvdc /dev/xvdd"   # must have spaces between and quoted, blank will ignore RAIDing drives
raid[1][mount_path]=/mnt/raid

raid[2][name]=MY_RAID
raid[2][target]=/dev/md0
raid[2][devices]="/dev/xvdc /dev/xvdd"   # must have spaces between and quoted, blank will ignore RAIDing drives
raid[2][mount_path]=/mnt/raid

## SSD Non-RAIDed drives
declare -A sdrives
ssds[1][name]=SSD
ssds[1][device]=/dev/xvdb        # blank = ignore
ssds[1][mount_path]=/mnt/ssd


## NOTE: for all settings below an empty value = ignore changing the setting
## See https://docs.datastax.com/en/datastax_enterprise/5.0/datastax_enterprise/install/installUnattendedDse.html
## for more information about what settings can be configured

# Cassandra installer settings
# Cassandra installer settings
declare -A ci
ci[cluster_name]=CHANGE_ME
## set to list of seeds separated by commas
ci[seeds]=${PUBLIC_IP}
ci[enable_vnodes]=1
ci[num_tokens]=64
# below unlikely to change
ci[update_system]=0
ci[data_dir]=${raid[1][mount_path]}/cassandra/data
ci[commitlog_dir]=${ssds[1][mount_path]}/cassandra/commitlog
ci[saved_caches_dir]=${raid[1][mount_path]}/cassandra/saved_caches

# enable which DSE features 0 = disable, 1 = enable
declare -A dsef
dsef[SOLR_ENABLED]=0
dsef[SPARK_ENABLED]=0
dsef[GRAPH_ENABLED]=0
dsef[HADOOP_ENABLED]=0
dsef[CFS_ENABLED]=0


# cassandra.yaml settings
declare -A cs
cs[listen_address]=${PRIVATE_IP}
cs[broadcast_rpc_address]=${PUBLIC_IP}
cs[rpc_address]=0.0.0.0
cs[endpoint_snitch]=GossipingPropertyFileSnitch
cs[phi_convict_threshold]=12   #always set to 12+ for AWS
## enable authorization
cs[authenticator]=com.datastax.bdp.cassandra.auth.DseAuthenticator
cs[authorizer]=com.datastax.bdp.cassandra.auth.DseAuthorizer
## Optimizations based on node setup
cs[memtable_flush_writers]=4
cs[memtable_cleanup_threshold]=0.6
cs[concurrent_reads]=64
cs[concurrent_writes]=32
cs[concurrent_compactors]=2
cs[compaction_throughput_mb_per_sec]=64
cs[hints_directory]=${raid[1][mount_path]}/cassandra/hints


# cassandra-rackdc.properties settings
declare -A cr
cr[dc]=${DC_NAME}
cr[rack]=${RACK}
cr[dc_suffix]=
cr[prefer_local]=

# cassandra-env.sh changes
declare -A cenv
cenv[MAX_HEAP_SIZE]=32G

# Datastax-Agent settings
declare -A dsa
dsa[local_interface]=${PUBLIC_IP}
# opcenter public IP address
dsa[stomp_address]=${PUBLIC_IP}


# dse.yaml settings blank = ignore
declare -A dseyaml
dseyaml[solr_data_dir]=${ssds[1][mount_path]}/solr.data


declare -A sysset
sysset[vm.dirty_background_bytes]=1048576   # 10mb
sysset[vm.dirty_bytes]=1073741824   # 1G
sysset[vm.swappiness]=1   # 1 or 0
sysset[vm.max_map_count]=1048576
sysset[fs.file-max]=1000000
sysset[vm.nr_hugepages]=0


### End config


if [ $OS_TYPE == "centos" ]; then
    INSTALLER_NAME=yum
else
    INSTALLER_NAME=apt-get
fi

#install xfs tools
echo "Installing xfs tools..."
sudo ${INSTALLER_NAME} install xfsprogs mdadm -y


echo "Creating raid devices..."
for i in "${!raid[@]}"
do
    RAID_SET=${raid[$i]}

    # raid[1][name]=MY_RAID
    # raid[1][target]=/dev/md0
    # raid[1][devices]="/dev/xvdc /dev/xvdd"   # must have spaces between and quoted, blank will ignore RAIDing drives
    # raid[1][mount_path]=/mnt/raid

    if [ "${RAID_SET[devices]}x" != "x" ]; then
        RAID_DEVICE_CNT=$(( $(echo ${RAID_SET[devices]} | tr -cd " " | wc -c) + 1 ))

        if [ $CLOUD == "azure" ]; then
            for d in $(echo ${RAID_SET[devices]} | tr -cd " ")
            do
                echo -e "o\nn\np\n1\n\n\nw" | fdisk $d
            done
        fi

        echo "Creating raid to ${RAID_SET[target]}..."
        mdadm --create ${RAID_SET[target]} --level=0 --name=${RAID_SET[name]} \
            --raid-devices=${RAID_DEVICE_CNT} ${RAID_SET[devices]}

        echo "Formatting raid device ${RAID_SET[target]}..."
        mkfs.xfs -L ${RAID_SET[name]} ${RAID_SET[target]}
        mkdir -p ${RAID_SET[mount_path]}

        echo "Mounting raid device at ${RAID_SET[mount_path]}..."
        mount -o discard LABEL=${RAID_SET[name]} ${RAID_SET[mount_path]}

        echo "Adding raid drive to fstab for restart safety..."
        if [ $CLOUD != "azure" ]; then
            echo "${RAID_SET[target]}   ${RAID_SET[mount_path]}   xfs    defaults,nofail,discard   0   2" | tee -a /etc/fstab
        else
            RAID_UUID=$(grep ${RAID_SET[target]} blkid | awk -F= '{print $2}' | awk -F'"' '{print $2}')
            echo "UUID=${RAID_UUID}   ${RAID_SET[mount_path]}   xfs    defaults,nofail,discard,nobarrier   0   2" | tee -a /etc/fstab
        fi

        RAID_BLOCK_DEVICE=$(echo ${RAID_SET[target]##*/})

        # do things for ssd best practices
        echo "Updating device with ssd best practices for ${RAID_BLOCK_DEVICE}..."
        echo 0 > /sys/block/${RAID_BLOCK_DEVICE}/queue/rotational
        echo 1 > /sys/block/${RAID_BLOCK_DEVICE}/queue/nomerges
        echo 8 > /sys/block/${RAID_BLOCK_DEVICE}/queue/read_ahead_kb
        echo deadline > /sys/block/${RAID_BLOCK_DEVICE}/queue/scheduler
        blockdev --setra 8 /dev/${RAID_BLOCK_DEVICE}
    fi
done

echo "Creating SSD devices..."
for i in "${!ssds[@]}"
do
    # ssds[1][name]=SSD
    # ssds[1][device]=/dev/xvdb        # blank = ignore
    # ssds[1][mount_path]=/mnt/ssd

    SSD_SET=${ssds[$i]}

    if [ "${SSD_SET[device]}x" != "x" ]; then

        # Format SSD to xfs
        echo "Formatting attached ssd to xfs..."
        mkfs.xfs -K ${SSD_SET[device]}

        echo "Creating mount directory ${SSD_SET[mount_path]}..."
        mkdir -p ${SSD_SET[mount_path]}

        # mount drive to proper location
        echo "Mounting the drive..."
        mount -o discard ${SSD_SET[device]} ${SSD_SET[mount_path]}

        echo "Adding ssd drive to fstab for restart safety..."
        echo "${SSD_SET[device]}   ${SSD_SET[mount_path]}   xfs    defaults,nofail,discard   0   2" | tee -a /etc/fstab

        SSD_BLOCK_DEVICE=$(echo ${SSD_SET[device]##*/})

        # do things for ssd best practices
        echo "Updating device with ssd best practices for ${SSD_BLOCK_DEVICE}..."
        echo 0 > /sys/block/${SSD_BLOCK_DEVICE}/queue/rotational
        echo 1 > /sys/block/${SSD_BLOCK_DEVICE}/queue/nomerges
        echo 8 > /sys/block/${SSD_BLOCK_DEVICE}/queue/read_ahead_kb
        echo deadline > /sys/block/${SSD_BLOCK_DEVICE}/queue/scheduler
        blockdev --setra 8 /dev/${SSD_BLOCK_DEVICE}
    fi
fi

#### start base.sh
echo "Updating system and base packages..."

if [ "${UPDATE_SYS_PACKAGES}" == "true" ]; then
    if [ $OS_TYPE != "centos" ]; then
        ${INSTALLER_NAME} update -y
        ${INSTALLER_NAME} upgrade -y
    else
        ${INSTALLER_NAME} update -y
    fi
fi

# disable huge page caches
if [ -d /sys/kernel/mm/transparent_hugepage ]; then
    echo "Disabling hugepages..."
    echo never > /sys/kernel/mm/transparent_hugepage/enabled
    echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi

echo "Installing default packages..."
${INSTALLER_NAME} install -y ${DEFAULT_APT_PKGS}

echo "Setting sysctl settings to be optimum..."
for i in "${!sysset[@]}"
do
    KEY=$i
    VALUE=${sysset[$i]}
    if [ "${VALUE}x" != "x" ]; then
        sysctl -w ${i}=${VALUE}
    fi
done

sysctl -p

#### install java
JAVA_VERSION=$(java -version 2>&1 >/dev/null | grep 'java version' | awk '{print $3}' | awk -F'"' '{print $2}')

if [[ $JAVA_VERSION != "1.8"* ]]; then
    echo "Installing Java 8..."

    if [ $OS_TYPE != "centos" ]; then
        add-apt-repository ppa:webupd8team/java -y > /dev/null
        aptitude update -y > /dev/null
        echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
        aptitude install oracle-java8-installer -y > /dev/null

        # make sure java8 is the version
        update-java-alternatives -s java-8-oracle
    else
        # JIC, remove old file
        rm -f ${INSTALL_DIR}/jdk*rpm
        wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \
    --directory-prefix=${INSTALL_DIR} "${RPM_JAVA_LINK}"
        JAVA_RPM=$(ls ${INSTALL_DIR}| grep jdk)
        yum localinstall ${INSTALL_DIR}/${JAVA_RPM} -y
    fi
else
    echo "Java 8 already installed skipping..."
fi

#### install DSE 5.0
echo "Installing DSE..."

DOWNLOAD_BASE_URL=http://downloads.datastax.com/enterprise
INSTALL_DIR=./installers
CONFIG_DIR=/etc/dse
DSE_DEFAULT=/etc/default/dse
DSE_YAML=${CONFIG_DIR}/dse.yaml
CASSANDRA_YAML=${CONFIG_DIR}/cassandra/cassandra.yaml
CASSANDRA_ENV=${CONFIG_DIR}/cassandra/cassandra-env.sh
CASSANDRA_RACK_DC_PROP=${CONFIG_DIR}/cassandra/cassandra-rackdc.properties

mkdir -p ${INSTALL_DIR}

if [ ! -f ${INSTALL_DIR}/${DSE_INSTALL_FILE} ]; then
    echo "Downloading DSE installer..."
    wget --directory-prefix=${INSTALL_DIR} \
        ${DOWNLOAD_BASE_URL}/DataStaxEnterprise-${DSE_VERSION}-linux-x64-installer.run \
        --user=${DSE_ACADEMY_USER} --password=${DSE_ACADEMY_PASS}
else
    # check if installer file is enough bytes
    DSE_INSTALL_MIN_SIZE=790000000
    DSE_ACTUAL_SIZE=$(wc -c <"$file")
    if [ $DSE_ACTUAL_SIZE -lt $DSE_INSTALL_MIN_SIZE ]; then
        echo "Found installer, but its below minimum size downloading again..."
        rm -f ${INSTALL_DIR}/${DSE_INSTALL_FILE}
        wget --directory-prefix=${INSTALL_DIR} \
            ${DOWNLOAD_BASE_URL}/DataStaxEnterprise-${DSE_VERSION}-linux-x64-installer.run \
            --user=${DSE_ACADEMY_USER} --password=${DSE_ACADEMY_PASS}
    else
        echo "DSE installer exists skipping download..."
    fi
fi

chmod +x ${INSTALL_DIR}/${DSE_INSTALL_FILE}

echo "Installing DSE..."
${INSTALL_DIR}/${DSE_INSTALL_FILE} --mode unattended --start_services 0 \
    --update_system ${ci[update_system]} \
    --seeds ${ci[seeds]} \
    --cassandra_data_dir ${ci[data_dir]} \
    --cassandra_commitlog_dir ${ci[commitlog_dir]} \
    --cassandra_saved_caches_dir ${ci[saved_caches_dir]} \
    --ring_name "${ci[cluster_name]}" \
    --enable_vnodes ${ci[enable_vnodes]} \
    --num_tokens ${ci[num_tokens]}

echo "Updating cassandra.yaml settings for current node..."
for i in "${!cs[@]}"
do
    KEY=$i
    VALUE=${cs[$i]}
    if [ "${VALUE}x" != "x" ]; then
        if [ $KEY == "authenticator" ] || [ $KEY == "authorizer" ] ; then
            sudo sed -i --follow-symlinks -e "s@\^${KEY}:\(.*\)@${KEY}: ${VALUE}@g" ${CASSANDRA_YAML}
        else
            sudo sed -i --follow-symlinks -e "s@\(^#${KEY}\|^${KEY}\|^# ${KEY}\): \(.*\)@${KEY}: ${VALUE}@g" ${CASSANDRA_YAML}
        fi
    fi
done

echo "Making sure cassandra directory exists and correct user..."
mkdir -p ${ci[data_dir]} && chown -R cassandra:cassandra ${ci[data_dir]}
mkdir -p ${ci[commitlog_dir]} && chown -R cassandra:cassandra ${ci[commitlog_dir]}
mkdir -p ${ci[saved_caches_dir]} && chown -R cassandra:cassandra ${ci[saved_caches_dir]}
mkdir -p ${cs[hints_directory]} && chown -R cassandra:cassandra ${cs[hints_directory]}

if [ "${dseyaml[solr_data_dir]}x" != "x" ]; then
    mkdir -p ${dseyaml[solr_data_dir]}
    chown cassandra:cassandra ${dseyaml[solr_data_dir]}
fi

echo "Updating cassandra-env.sh for current node..."
for i in "${!cenv[@]}"
do
    KEY=$i
    VALUE=${cenv[$i]}
    if [ "${VALUE}x" != "x" ]; then
        sed -i --follow-symlinks -e "s@^#${KEY}=\(.*\)@${KEY}=\"${VALUE}\"@g" ${CASSANDRA_ENV}
    fi
done

echo "Updating cassandra-rackdc.properties for current node..."
for i in "${!cr[@]}"
do
    KEY=$i
    VALUE=${cr[$i]}
    if [ "${VALUE}x" != "x" ]; then
        sed -i --follow-symlinks -e "s@\(^#${KEY}\|^${KEY}\|^# ${KEY}\)=\(.*\)@${KEY}=${VALUE}@g" ${CASSANDRA_RACK_DC_PROP}
    fi
done

echo "Updating dse.yaml for current node..."
for i in "${!dseyaml[@]}"
do
    KEY=$i
    VALUE=${dseyaml[$i]}
    if [ "${VALUE}x" != "x" ]; then
        sed -i --follow-symlinks -e "s@\(^#${KEY}\|^${KEY}\|^# ${KEY}\):\(.*\)@${KEY}: ${VALUE}@g" ${DSE_YAML}
    fi
done

echo "Enabling DSE services..."
for i in "${!dsef[@]}"
do
    KEY=$i
    VALUE=${dsef[$i]}
    if [ "${VALUE}x" != "x" ]; then
        sed -i --follow-symlinks -e "s@\(^${KEY}\)=\(.*\)@${KEY}=${VALUE}@g" ${DSE_DEFAULT}
    fi
done

#### start install agent
echo "Installing Datastax Agent..."

DATASTAX_AGENT_CONF_DIR=/var/lib/datastax-agent/conf/
DATASTAX_AGENT_CONF_FILE=${DATASTAX_AGENT_CONF_DIR}/address.yaml

if [ $OS_TYPE == "centos" ]; then

    wget --directory-prefix=${INSTALL_DIR} \
        ${DOWNLOAD_BASE_URL}/opscenter-${OPSC_VERSION}.tar.gz \
        --user=${DSE_ACADEMY_USER} --password=${DSE_ACADEMY_PASS}
    tar -xf ${INSTALL_DIR}/opscenter-${OPSC_VERSION}.tar.gz -C ${INSTALL_DIR}/

    AGENT_RPM=${INSTALL_DIR}/opscenter-${OPSC_VERSION}/agent/datastax-agent.rpm
    yum localinstall ${INSTALL_DIR}/${AGENT_RPM} -y
else
    echo "deb http://debian.datastax.com/community stable main" | tee -a /etc/apt/sources.list.d/datastax.community.list

    curl -L https://debian.datastax.com/debian/repo_key | sudo apt-key add -

    sudo apt-get update
    sudo apt-get install datastax-agent
fi

# make sure conf folder exists
mkdir -p ${DATASTAX_AGENT_CONF_DIR}

echo "Updating datastax-agent address.yaml conf for current node..."
for i in "${!dsa[@]}"
do
    KEY=$i
    VALUE=${dsa[$i]}
    if [ "${VALUE}x" != "x" ]; then
        echo "${KEY}: ${VALUE}" | tee -a ${DATASTAX_AGENT_CONF_FILE}
    fi
done

echo "Starting datastax-agent..."
service datastax-agent start

if [ $OS_TYPE != "centos" ]; then
    service ntp start
else
    chkconfig ntpd on
    if [ -f /etc/init.d/ntpd ]; then
        /etc/init.d/ntpd start
    else
        service ntpd start
    fi
fi

echo "Updating current time..."
ntpdate pool.ntp.org

echo "Time to check and start DSE. If everything is OK files in ${INSTALL_DIR} can be deleted."
	#!/bin/bash

	########################################################################
	### ###
	### Install Script to install DSE 5.0 on Ubuntu or CentOS ###
	### created by Brad Vernon brad.vernon@datastax.com ###
	### ###
	########################################################################

	# AWS Instance gathering, do not delete
	getAwsMetaData() {
	METADATA_URL_BASE="http://169.254.169.254/latest/meta-data"
	PRIVATE_IP=$(curl --silent ${METADATA_URL_BASE}/local-ipv4)
	PUBLIC_IP=$(curl --silent ${METADATA_URL_BASE}/public-ipv4)
	DC_AND_AZ=$(curl --silent ${METADATA_URL_BASE}/placement/availability-zone)
	DC_AND_AZ_SPLIT=(${DC_AND_AZ//-/ })
	AZ=${DC_AND_AZ_SPLIT[${#DC_AND_AZ_SPLIT[@]} - 1]}
	REGION_NUM=${AZ:0:1}
	DC_NAME=$(echo ${DC_AND_AZ} \| rev \| cut -c 2- \| rev)
	RACK=$(echo ${DC_AND_AZ} \| rev \| cut -c -2 \| rev)
	}

	getAzureDc() {
	IP=$1
	if [ IP == "10.1."* ]; then
	DC=us-west
	else
	DC=us-east
	fi
	}

	#Azure install info
	getAzureMetaData() {
	METADATA_URL="http://169.254.169.254/metadata/v1/InstanceInfo"
	PRIVATE_IP=$(ifconfig eth0 \| grep inet \| awk '{print $2}' \| sed 's/addr://' \| grep .)
	PUBLIC_IP=$(curl --silent ipinfo.io/ip)
	NODE_INFO=$(curl --silent ${METADATA_URL_BASE})
	RACK=$(echo $NODE_INFO \| awk -F":" '{print $4}' \| awk -F"}" '{print $1}')
	}

	### Start config

	CLOUD=azure # options: azure, aws
	OS_TYPE=centos # options: centos, ubuntu (Note: RH = centos)
	UPDATE_SYS_PACKAGES=true
	DEFAULT_APT_PKGS="vim sysstat wget htop python-pip python-dev libev4 libev-dev ntp"

	# Username and pass for academy to download installer
	DSE_ACADEMY_USER=
	DSE_ACADEMY_PASS=
	# DSE and Opscenter Agent Versions
	DSE_VERSION=5.0.5
	OPSC_VERSION=6.0.7

	## Latest version visit http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html
	RPM_JAVA_LINK=http://download.oracle.com/otn-pub/java/jdk/8u112-b15/jdk-8u112-linux-x64.rpm


	## drives
	## RAIDed drives can be EBS or local SSDs
	declare -A raid
	raid[1][name]=MY_RAID
	raid[1][target]=/dev/md0
	raid[1][devices]="/dev/xvdc /dev/xvdd" # must have spaces between and quoted, blank will ignore RAIDing drives
	raid[1][mount_path]=/mnt/raid

	raid[2][name]=MY_RAID
	raid[2][target]=/dev/md0
	raid[2][devices]="/dev/xvdc /dev/xvdd" # must have spaces between and quoted, blank will ignore RAIDing drives
	raid[2][mount_path]=/mnt/raid

	## SSD Non-RAIDed drives
	declare -A sdrives
	ssds[1][name]=SSD
	ssds[1][device]=/dev/xvdb # blank = ignore
	ssds[1][mount_path]=/mnt/ssd


	## NOTE: for all settings below an empty value = ignore changing the setting
	## See https://docs.datastax.com/en/datastax_enterprise/5.0/datastax_enterprise/install/installUnattendedDse.html
	## for more information about what settings can be configured

	# Cassandra installer settings
	# Cassandra installer settings
	declare -A ci
	ci[cluster_name]=CHANGE_ME
	## set to list of seeds separated by commas
	ci[seeds]=${PUBLIC_IP}
	ci[enable_vnodes]=1
	ci[num_tokens]=64
	# below unlikely to change
	ci[update_system]=0
	ci[data_dir]=${raid[1][mount_path]}/cassandra/data
	ci[commitlog_dir]=${ssds[1][mount_path]}/cassandra/commitlog
	ci[saved_caches_dir]=${raid[1][mount_path]}/cassandra/saved_caches

	# enable which DSE features 0 = disable, 1 = enable
	declare -A dsef
	dsef[SOLR_ENABLED]=0
	dsef[SPARK_ENABLED]=0
	dsef[GRAPH_ENABLED]=0
	dsef[HADOOP_ENABLED]=0
	dsef[CFS_ENABLED]=0


	# cassandra.yaml settings
	declare -A cs
	cs[listen_address]=${PRIVATE_IP}
	cs[broadcast_rpc_address]=${PUBLIC_IP}
	cs[rpc_address]=0.0.0.0
	cs[endpoint_snitch]=GossipingPropertyFileSnitch
	cs[phi_convict_threshold]=12 #always set to 12+ for AWS
	## enable authorization
	cs[authenticator]=com.datastax.bdp.cassandra.auth.DseAuthenticator
	cs[authorizer]=com.datastax.bdp.cassandra.auth.DseAuthorizer
	## Optimizations based on node setup
	cs[memtable_flush_writers]=4
	cs[memtable_cleanup_threshold]=0.6
	cs[concurrent_reads]=64
	cs[concurrent_writes]=32
	cs[concurrent_compactors]=2
	cs[compaction_throughput_mb_per_sec]=64
	cs[hints_directory]=${raid[1][mount_path]}/cassandra/hints


	# cassandra-rackdc.properties settings
	declare -A cr
	cr[dc]=${DC_NAME}
	cr[rack]=${RACK}
	cr[dc_suffix]=
	cr[prefer_local]=

	# cassandra-env.sh changes
	declare -A cenv
	cenv[MAX_HEAP_SIZE]=32G

	# Datastax-Agent settings
	declare -A dsa
	dsa[local_interface]=${PUBLIC_IP}
	# opcenter public IP address
	dsa[stomp_address]=${PUBLIC_IP}


	# dse.yaml settings blank = ignore
	declare -A dseyaml
	dseyaml[solr_data_dir]=${ssds[1][mount_path]}/solr.data


	declare -A sysset
	sysset[vm.dirty_background_bytes]=1048576 # 10mb
	sysset[vm.dirty_bytes]=1073741824 # 1G
	sysset[vm.swappiness]=1 # 1 or 0
	sysset[vm.max_map_count]=1048576
	sysset[fs.file-max]=1000000
	sysset[vm.nr_hugepages]=0


	### End config


	if [ $OS_TYPE == "centos" ]; then
	INSTALLER_NAME=yum
	else
	INSTALLER_NAME=apt-get
	fi

	#install xfs tools
	echo "Installing xfs tools..."
	sudo ${INSTALLER_NAME} install xfsprogs mdadm -y


	echo "Creating raid devices..."
	for i in "${!raid[@]}"
	do
	RAID_SET=${raid[$i]}

	# raid[1][name]=MY_RAID
	# raid[1][target]=/dev/md0
	# raid[1][devices]="/dev/xvdc /dev/xvdd" # must have spaces between and quoted, blank will ignore RAIDing drives
	# raid[1][mount_path]=/mnt/raid

	if [ "${RAID_SET[devices]}x" != "x" ]; then
	RAID_DEVICE_CNT=$(( $(echo ${RAID_SET[devices]} \| tr -cd " " \| wc -c) + 1 ))

	if [ $CLOUD == "azure" ]; then
	for d in $(echo ${RAID_SET[devices]} \| tr -cd " ")
	do
	echo -e "o\nn\np\n1\n\n\nw" \| fdisk $d
	done
	fi

	echo "Creating raid to ${RAID_SET[target]}..."
	mdadm --create ${RAID_SET[target]} --level=0 --name=${RAID_SET[name]} \
	--raid-devices=${RAID_DEVICE_CNT} ${RAID_SET[devices]}

	echo "Formatting raid device ${RAID_SET[target]}..."
	mkfs.xfs -L ${RAID_SET[name]} ${RAID_SET[target]}
	mkdir -p ${RAID_SET[mount_path]}

	echo "Mounting raid device at ${RAID_SET[mount_path]}..."
	mount -o discard LABEL=${RAID_SET[name]} ${RAID_SET[mount_path]}

	echo "Adding raid drive to fstab for restart safety..."
	if [ $CLOUD != "azure" ]; then
	echo "${RAID_SET[target]} ${RAID_SET[mount_path]} xfs defaults,nofail,discard 0 2" \| tee -a /etc/fstab
	else
	RAID_UUID=$(grep ${RAID_SET[target]} blkid \| awk -F= '{print $2}' \| awk -F'"' '{print $2}')
	echo "UUID=${RAID_UUID} ${RAID_SET[mount_path]} xfs defaults,nofail,discard,nobarrier 0 2" \| tee -a /etc/fstab
	fi

	RAID_BLOCK_DEVICE=$(echo ${RAID_SET[target]##*/})

	# do things for ssd best practices
	echo "Updating device with ssd best practices for ${RAID_BLOCK_DEVICE}..."
	echo 0 > /sys/block/${RAID_BLOCK_DEVICE}/queue/rotational
	echo 1 > /sys/block/${RAID_BLOCK_DEVICE}/queue/nomerges
	echo 8 > /sys/block/${RAID_BLOCK_DEVICE}/queue/read_ahead_kb
	echo deadline > /sys/block/${RAID_BLOCK_DEVICE}/queue/scheduler
	blockdev --setra 8 /dev/${RAID_BLOCK_DEVICE}
	fi
	done

	echo "Creating SSD devices..."
	for i in "${!ssds[@]}"
	do
	# ssds[1][name]=SSD
	# ssds[1][device]=/dev/xvdb # blank = ignore
	# ssds[1][mount_path]=/mnt/ssd

	SSD_SET=${ssds[$i]}

	if [ "${SSD_SET[device]}x" != "x" ]; then

	# Format SSD to xfs
	echo "Formatting attached ssd to xfs..."
	mkfs.xfs -K ${SSD_SET[device]}

	echo "Creating mount directory ${SSD_SET[mount_path]}..."
	mkdir -p ${SSD_SET[mount_path]}

	# mount drive to proper location
	echo "Mounting the drive..."
	mount -o discard ${SSD_SET[device]} ${SSD_SET[mount_path]}

	echo "Adding ssd drive to fstab for restart safety..."
	echo "${SSD_SET[device]} ${SSD_SET[mount_path]} xfs defaults,nofail,discard 0 2" \| tee -a /etc/fstab

	SSD_BLOCK_DEVICE=$(echo ${SSD_SET[device]##*/})

	# do things for ssd best practices
	echo "Updating device with ssd best practices for ${SSD_BLOCK_DEVICE}..."
	echo 0 > /sys/block/${SSD_BLOCK_DEVICE}/queue/rotational
	echo 1 > /sys/block/${SSD_BLOCK_DEVICE}/queue/nomerges
	echo 8 > /sys/block/${SSD_BLOCK_DEVICE}/queue/read_ahead_kb
	echo deadline > /sys/block/${SSD_BLOCK_DEVICE}/queue/scheduler
	blockdev --setra 8 /dev/${SSD_BLOCK_DEVICE}
	fi
	fi

	#### start base.sh
	echo "Updating system and base packages..."

	if [ "${UPDATE_SYS_PACKAGES}" == "true" ]; then
	if [ $OS_TYPE != "centos" ]; then
	${INSTALLER_NAME} update -y
	${INSTALLER_NAME} upgrade -y
	else
	${INSTALLER_NAME} update -y
	fi
	fi

	# disable huge page caches
	if [ -d /sys/kernel/mm/transparent_hugepage ]; then
	echo "Disabling hugepages..."
	echo never > /sys/kernel/mm/transparent_hugepage/enabled
	echo never > /sys/kernel/mm/transparent_hugepage/defrag
	fi

	echo "Installing default packages..."
	${INSTALLER_NAME} install -y ${DEFAULT_APT_PKGS}

	echo "Setting sysctl settings to be optimum..."
	for i in "${!sysset[@]}"
	do
	KEY=$i
	VALUE=${sysset[$i]}
	if [ "${VALUE}x" != "x" ]; then
	sysctl -w ${i}=${VALUE}
	fi
	done

	sysctl -p

	#### install java
	JAVA_VERSION=$(java -version 2>&1 >/dev/null \| grep 'java version' \| awk '{print $3}' \| awk -F'"' '{print $2}')

	if [[ $JAVA_VERSION != "1.8"* ]]; then
	echo "Installing Java 8..."

	if [ $OS_TYPE != "centos" ]; then
	add-apt-repository ppa:webupd8team/java -y > /dev/null
	aptitude update -y > /dev/null
	echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true \| sudo /usr/bin/debconf-set-selections
	aptitude install oracle-java8-installer -y > /dev/null

	# make sure java8 is the version
	update-java-alternatives -s java-8-oracle
	else
	# JIC, remove old file
	rm -f ${INSTALL_DIR}/jdk*rpm
	wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" \
	--directory-prefix=${INSTALL_DIR} "${RPM_JAVA_LINK}"
	JAVA_RPM=$(ls ${INSTALL_DIR}\| grep jdk)
	yum localinstall ${INSTALL_DIR}/${JAVA_RPM} -y
	fi
	else
	echo "Java 8 already installed skipping..."
	fi

	#### install DSE 5.0
	echo "Installing DSE..."

	DOWNLOAD_BASE_URL=http://downloads.datastax.com/enterprise
	INSTALL_DIR=./installers
	CONFIG_DIR=/etc/dse
	DSE_DEFAULT=/etc/default/dse
	DSE_YAML=${CONFIG_DIR}/dse.yaml
	CASSANDRA_YAML=${CONFIG_DIR}/cassandra/cassandra.yaml
	CASSANDRA_ENV=${CONFIG_DIR}/cassandra/cassandra-env.sh
	CASSANDRA_RACK_DC_PROP=${CONFIG_DIR}/cassandra/cassandra-rackdc.properties

	mkdir -p ${INSTALL_DIR}

	if [ ! -f ${INSTALL_DIR}/${DSE_INSTALL_FILE} ]; then
	echo "Downloading DSE installer..."
	wget --directory-prefix=${INSTALL_DIR} \
	${DOWNLOAD_BASE_URL}/DataStaxEnterprise-${DSE_VERSION}-linux-x64-installer.run \
	--user=${DSE_ACADEMY_USER} --password=${DSE_ACADEMY_PASS}
	else
	# check if installer file is enough bytes
	DSE_INSTALL_MIN_SIZE=790000000
	DSE_ACTUAL_SIZE=$(wc -c <"$file")
	if [ $DSE_ACTUAL_SIZE -lt $DSE_INSTALL_MIN_SIZE ]; then
	echo "Found installer, but its below minimum size downloading again..."
	rm -f ${INSTALL_DIR}/${DSE_INSTALL_FILE}
	wget --directory-prefix=${INSTALL_DIR} \
	${DOWNLOAD_BASE_URL}/DataStaxEnterprise-${DSE_VERSION}-linux-x64-installer.run \
	--user=${DSE_ACADEMY_USER} --password=${DSE_ACADEMY_PASS}
	else
	echo "DSE installer exists skipping download..."
	fi
	fi

	chmod +x ${INSTALL_DIR}/${DSE_INSTALL_FILE}

	echo "Installing DSE..."
	${INSTALL_DIR}/${DSE_INSTALL_FILE} --mode unattended --start_services 0 \
	--update_system ${ci[update_system]} \
	--seeds ${ci[seeds]} \
	--cassandra_data_dir ${ci[data_dir]} \
	--cassandra_commitlog_dir ${ci[commitlog_dir]} \
	--cassandra_saved_caches_dir ${ci[saved_caches_dir]} \
	--ring_name "${ci[cluster_name]}" \
	--enable_vnodes ${ci[enable_vnodes]} \
	--num_tokens ${ci[num_tokens]}

	echo "Updating cassandra.yaml settings for current node..."
	for i in "${!cs[@]}"
	do
	KEY=$i
	VALUE=${cs[$i]}
	if [ "${VALUE}x" != "x" ]; then
	if [ $KEY == "authenticator" ] \|\| [ $KEY == "authorizer" ] ; then
	sudo sed -i --follow-symlinks -e "s@\^${KEY}:\(.*\)@${KEY}: ${VALUE}@g" ${CASSANDRA_YAML}
	else
	sudo sed -i --follow-symlinks -e "s@\(^#${KEY}\\|^${KEY}\\|^# ${KEY}\): \(.*\)@${KEY}: ${VALUE}@g" ${CASSANDRA_YAML}
	fi
	fi
	done

	echo "Making sure cassandra directory exists and correct user..."
	mkdir -p ${ci[data_dir]} && chown -R cassandra:cassandra ${ci[data_dir]}
	mkdir -p ${ci[commitlog_dir]} && chown -R cassandra:cassandra ${ci[commitlog_dir]}
	mkdir -p ${ci[saved_caches_dir]} && chown -R cassandra:cassandra ${ci[saved_caches_dir]}
	mkdir -p ${cs[hints_directory]} && chown -R cassandra:cassandra ${cs[hints_directory]}

	if [ "${dseyaml[solr_data_dir]}x" != "x" ]; then
	mkdir -p ${dseyaml[solr_data_dir]}
	chown cassandra:cassandra ${dseyaml[solr_data_dir]}
	fi

	echo "Updating cassandra-env.sh for current node..."
	for i in "${!cenv[@]}"
	do
	KEY=$i
	VALUE=${cenv[$i]}
	if [ "${VALUE}x" != "x" ]; then
	sed -i --follow-symlinks -e "s@^#${KEY}=\(.*\)@${KEY}=\"${VALUE}\"@g" ${CASSANDRA_ENV}
	fi
	done

	echo "Updating cassandra-rackdc.properties for current node..."
	for i in "${!cr[@]}"
	do
	KEY=$i
	VALUE=${cr[$i]}
	if [ "${VALUE}x" != "x" ]; then
	sed -i --follow-symlinks -e "s@\(^#${KEY}\\|^${KEY}\\|^# ${KEY}\)=\(.*\)@${KEY}=${VALUE}@g" ${CASSANDRA_RACK_DC_PROP}
	fi
	done

	echo "Updating dse.yaml for current node..."
	for i in "${!dseyaml[@]}"
	do
	KEY=$i
	VALUE=${dseyaml[$i]}
	if [ "${VALUE}x" != "x" ]; then
	sed -i --follow-symlinks -e "s@\(^#${KEY}\\|^${KEY}\\|^# ${KEY}\):\(.*\)@${KEY}: ${VALUE}@g" ${DSE_YAML}
	fi
	done

	echo "Enabling DSE services..."
	for i in "${!dsef[@]}"
	do
	KEY=$i
	VALUE=${dsef[$i]}
	if [ "${VALUE}x" != "x" ]; then
	sed -i --follow-symlinks -e "s@\(^${KEY}\)=\(.*\)@${KEY}=${VALUE}@g" ${DSE_DEFAULT}
	fi
	done

	#### start install agent
	echo "Installing Datastax Agent..."

	DATASTAX_AGENT_CONF_DIR=/var/lib/datastax-agent/conf/
	DATASTAX_AGENT_CONF_FILE=${DATASTAX_AGENT_CONF_DIR}/address.yaml

	if [ $OS_TYPE == "centos" ]; then

	wget --directory-prefix=${INSTALL_DIR} \
	${DOWNLOAD_BASE_URL}/opscenter-${OPSC_VERSION}.tar.gz \
	--user=${DSE_ACADEMY_USER} --password=${DSE_ACADEMY_PASS}
	tar -xf ${INSTALL_DIR}/opscenter-${OPSC_VERSION}.tar.gz -C ${INSTALL_DIR}/

	AGENT_RPM=${INSTALL_DIR}/opscenter-${OPSC_VERSION}/agent/datastax-agent.rpm
	yum localinstall ${INSTALL_DIR}/${AGENT_RPM} -y
	else
	echo "deb http://debian.datastax.com/community stable main" \| tee -a /etc/apt/sources.list.d/datastax.community.list

	curl -L https://debian.datastax.com/debian/repo_key \| sudo apt-key add -

	sudo apt-get update
	sudo apt-get install datastax-agent
	fi

	# make sure conf folder exists
	mkdir -p ${DATASTAX_AGENT_CONF_DIR}

	echo "Updating datastax-agent address.yaml conf for current node..."
	for i in "${!dsa[@]}"
	do
	KEY=$i
	VALUE=${dsa[$i]}
	if [ "${VALUE}x" != "x" ]; then
	echo "${KEY}: ${VALUE}" \| tee -a ${DATASTAX_AGENT_CONF_FILE}
	fi
	done

	echo "Starting datastax-agent..."
	service datastax-agent start

	if [ $OS_TYPE != "centos" ]; then
	service ntp start
	else
	chkconfig ntpd on
	if [ -f /etc/init.d/ntpd ]; then
	/etc/init.d/ntpd start
	else
	service ntpd start
	fi
	fi

	echo "Updating current time..."
	ntpdate pool.ntp.org

	echo "Time to check and start DSE. If everything is OK files in ${INSTALL_DIR} can be deleted."