Skip to content

Instantly share code, notes, and snippets.

@javagrails
Forked from nddipiazza/setup-fusion-cluster.sh
Created August 18, 2021 18:39
Show Gist options
  • Save javagrails/c121bad9a9edea60d96187e17f6a1475 to your computer and use it in GitHub Desktop.
Save javagrails/c121bad9a9edea60d96187e17f6a1475 to your computer and use it in GitHub Desktop.
Install a Fusion cluster
# Run this script to install a Fusion cluster locally.
#
# In the working directory you are in, it will create fusion-1, fusion-2, etc... directories.
#
# You will then take those directories and either run them from the same machine, or you can copy the directories to separate instances.
#
# There are two optional command line properties:
#
# --no-download Do not download Fusion from https://download.lucidworks.com instead use the tar.gz file in this directory already.
# -v Verbose mode.
#
# You must have an odd number of zookeeper hosts.
# If you are installing this cluster on the same host, make sure each fusion_*_ports entry is unique. Otherwise,
# you can have them the same.
# Enter the Fusion version here
fusion_version=3.1.1
# Enter each zkhost in a comma separated list. Must be an odd number of zkHosts. E.g. zkhost=241.1.0.20:9983,241.1.0.21:9983,241.1.0.22:9983
zkhost=127.0.0.1:9983
# For each node, enter a port for Fusion for all the different services. For same-box installation, each port must be unique as they are below. For a cluster where each node is on a different host, you should just use the same ports.
# For example fusion_ui_ports=8764,8764,8764 would be ok if you had a 3 node cluster on separate machines.
fusion_ui_ports=8764,8864
fusion_solr_ports=8983,9083
fusion_connectors_ports=8984,9084
fusion_api_ports=8765,8865
fusion_ui_stop_ports=7764,7864
fusion_solr_stop_ports=7983,8083
fusion_connectors_stop_ports=7984,8084
fusion_api_stop_ports=7765,7865
fusion_agent_ports=8091,8191
# No edits needed from this point on.
no_download=false
if [[ $@ == **--no-download** ]]
then
no_download=true
fi
verbose=''
if [[ $@ == **-v** ]]
then
verbose='v'
fi
if [ "$no_download" = false ] ; then
rm -f fusion-${fusion_version}.tar.gz
wget -O fusion-${fusion_version}.tar.gz https://download.lucidworks.com/fusion-${fusion_version}/fusion-${fusion_version}.tar.gz
fi
IFS=',' read -r -a fusion_ui_ports_array <<< ${fusion_ui_ports}
IFS=',' read -r -a fusion_solr_ports_array <<< ${fusion_solr_ports}
IFS=',' read -r -a fusion_connectors_ports_array <<< ${fusion_connectors_ports}
IFS=',' read -r -a fusion_api_ports_array <<< ${fusion_api_ports}
IFS=',' read -r -a fusion_ui_stop_ports_array <<< ${fusion_ui_stop_ports}
IFS=',' read -r -a fusion_solr_stop_ports_array <<< ${fusion_solr_stop_ports}
IFS=',' read -r -a fusion_connectors_stop_ports_array <<< ${fusion_connectors_stop_ports}
IFS=',' read -r -a fusion_api_stop_ports_array <<< ${fusion_api_stop_ports}
IFS=',' read -r -a fusion_agent_ports_array <<< ${fusion_agent_ports}
IFS=',' read -r -a zkarray <<< "$zkhost"
num_zk_nodes=${#zkarray[@]}
if [ $((num_zk_nodes%2)) -eq 0 ];
then
echo ERROR: Zookeeper must have odd number of nodes.
exit 1
else
echo "There are ${num_zk_nodes} zookeeper nodes"
fi
num_fusion_nodes=${#fusion_ui_ports_array[@]}
echo "There are ${num_fusion_nodes} fusion nodes"
i=0; while [ $i -lt ${num_fusion_nodes} ]; do
myzkid=$(($i + 1))
echo "Extracting fusion-${fusion_version}.tar.gz to fusion-${myzkid}"
rm -rf fusion-${myzkid}
mkdir fusion-${myzkid} && tar -xz${verbose}f fusion-${fusion_version}.tar.gz -C fusion-${myzkid} --strip-components 1
fusion_dir=fusion-${myzkid}/${fusion_version}
mkdir "$fusion_dir/data/zookeeper"
echo "$myzkid" > "$fusion_dir/data/zookeeper/myid"
echo "" >> "$fusion_dir/conf/fusion.properties"
echo "" >> "$fusion_dir/conf/fusion.properties"
echo "solr.zk.connect = $zkhost" >> "$fusion_dir/conf/fusion.properties"
echo "api.zk.connect = $zkhost" >> "$fusion_dir/conf/fusion.properties"
echo "connectors.zk.connect = $zkhost" >> "$fusion_dir/conf/fusion.properties"
echo "spark-master.zk.connect = $zkhost" >> "$fusion_dir/conf/fusion.properties"
echo "spark-worker.zk.connect = $zkhost" >> "$fusion_dir/conf/fusion.properties"
echo "ui.zk.connect = $zkhost" >> "$fusion_dir/conf/fusion.properties"
sed -i "s/# default.zk.connect = .*/default.zk.connect = $zkhost/g" "$fusion_dir/conf/fusion.properties"
sed -i "s/# default.address = .*/default.address = 127.0.0.1/g" "$fusion_dir/conf/fusion.properties"
if [[ $i -lt ${num_zk_nodes} ]]
then
echo "Need zk on host ${myzkid}"
IFS=':' read -r -a thiszkhostportarray <<< ${zkarray[$i]}
zkh=${thiszkhostportarray[0]}
zkp=${thiszkhostportarray[1]}
sed -i "s/zookeeper.port = .*/zookeeper.port = $zkp/g" "$fusion_dir/conf/fusion.properties"
else
echo "Do not need zk on host ${myzkid}"
sed -i "s/group.default = zookeeper, solr, api, connectors, ui/group.default = solr, api, connectors, ui/g" "$fusion_dir/conf/fusion.properties"
fi
sed -i "s/ui.port = .*/ui.port = ${fusion_ui_ports_array[$i]}/g" "$fusion_dir/conf/fusion.properties"
sed -i "s/solr.port = .*/solr.port = ${fusion_solr_ports_array[$i]}/g" "$fusion_dir/conf/fusion.properties"
sed -i "s/connectors.port = .*/connectors.port = ${fusion_connectors_ports_array[$i]}/g" "$fusion_dir/conf/fusion.properties"
sed -i "s/api.port = .*/api.port = ${fusion_api_ports_array[$i]}/g" "$fusion_dir/conf/fusion.properties"
sed -i "s/ui.stopPort = .*/ui.stopPort = ${fusion_ui_stop_ports_array[$i]}/g" "$fusion_dir/conf/fusion.properties"
sed -i "s/solr.stopPort = .*/solr.stopPort = ${fusion_solr_stop_ports_array[$i]}/g" "$fusion_dir/conf/fusion.properties"
sed -i "s/connectors.stopPort = .*/connectors.stopPort = ${fusion_connectors_stop_ports_array[$i]}/g" "$fusion_dir/conf/fusion.properties"
sed -i "s/api.stopPort = .*/api.stopPort = ${fusion_api_stop_ports_array[$i]}/g" "$fusion_dir/conf/fusion.properties"
sed -i "s/agent.port = .*/agent.port = ${fusion_agent_ports_array[$i]}/g" "$fusion_dir/conf/fusion.properties"
j=0; while [ $j -lt ${num_zk_nodes} ]; do
nextzkid=$((j+1))
IFS=':' read -r -a nextzkhostportarray <<< ${zkarray[$j]}
innerzkh=${nextzkhostportarray[0]}
echo "server.${nextzkid}=${innerzkh}:2888:3888" >> "$fusion_dir/conf/zookeeper/zoo.cfg"
j=$(($j + 1))
done
echo "Updated ${fusion_dir} with Zookeeper node ID=${myzkid},Host=${zkh},Port=${zkp}, Ui Port=${fusion_ui_ports_array[$i]}/StopPort=${fusion_ui_stop_ports_array[$i]}, Api Port=${fusion_api_ports_array[$i]}/StopPort=${fusion_api_stop_ports_array[$i]}, Connectors Port=${fusion_connectors_ports_array[$i]}/StopPort=${fusion_connectors_stop_ports_array[$i]}, Solr Port = ${fusion_solr_ports_array[$i]}/StopPort=${fusion_solr_stop_ports_array[$i]}"
i=$(($i + 1))
done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment