install-transformer.sh

#!/usr/bin/env bash

XF_VER=3.14.0
XF_ROOT=/opt
XF_HOME=$XF_ROOT/streamsets-transformer
XF_CONF=/etc/transformer
XF_LOG=/var/log/transformer
XF_DATA=/var/lib/transformer
XF_RESOURCES=/var/lib/transformer-resources
SPARK_VER=2.4.5
SPARK_ROOT=/opt
SPARK_HOME=$SPARK_ROOT/spark-$SPARK_VER
SPARK_EVENTS=/tmp/spark-events

echo "Creating installation directories"
sudo mkdir -p $XF_ROOT $XF_CONF $XF_LOG $XF_DATA $XF_RESOURCES $SPARK_ROOT

echo "Installing dependencies"
sudo apt-get -y -qq update > /dev/null
sudo apt-get -y -qq install openjdk-8-jdk-headless scala wget ntp > /dev/null

echo "Downloading Transformer package"
wget -qO - https://archives.streamsets.com/transformer/$XF_VER/tarball/streamsets-transformer-all-$XF_VER.tgz | sudo tar xzf - -C $XF_ROOT

echo "Downloading Spark package"
#wget -qO - http://apache.mirrors.tds.net/spark/spark-$SPARK_VER/spark-$SPARK_VER-bin-without-hadoop.tgz | sudo tar xzf - -C $SPARK_ROOT
wget -qO - http://apache.mirrors.tds.net/spark/spark-$SPARK_VER/spark-$SPARK_VER-bin-hadoop2.7.tgz | sudo tar xzf - -C $SPARK_ROOT

echo "Renaming Transformer download directory to $XF_HOME"
sudo mv $XF_HOME-$XF_VER $XF_HOME

echo "Renaming Spark download directory to $SPARK_HOME"
sudo mv $SPARK_ROOT/spark-$SPARK_VER-bin-hadoop2.7 $SPARK_HOME

echo "Installing config files to $XF_CONF"
sudo cp -R $XF_HOME/etc/* $XF_CONF

echo -e "Configuring Spark logging"
sudo mkdir -p $SPARK_HOME/logs
sudo mkdir -p $SPARK_EVENTS

echo "Creating transformer user"
sudo groupadd -r transformer && sudo useradd -r -d $XF_HOME -g transformer -s /sbin/nologin transformer

echo "Setting transformer user permissions"
sudo chown -R transformer:transformer $XF_HOME $XF_CONF $XF_LOG $XF_DATA $XF_RESOURCES $SPARK_HOME $SPARK_EVENTS

echo "Updating Transformer configuration settings..."
echo -e "\tUpdating service directories"
sudo sed -i "s@/opt/streamsets-transformer@$XF_HOME@" $XF_HOME/systemd/transformer.service
sudo sed -i "s@/etc/transformer@$XF_CONF@" $XF_HOME/systemd/transformer.service
sudo sed -i "s@/var/log\/transformer@$XF_LOG@" $XF_HOME/systemd/transformer.service
sudo sed -i "s@/var/lib/transformer@$XF_DATA@" $XF_HOME/systemd/transformer.service

echo -e "\tEnabling HTTPS"
sudo sed -i 's/http.port=19630/http.port=-1/' $XF_CONF/transformer.properties
sudo sed -i 's/https.port=-1/https.port=19630/' $XF_CONF/transformer.properties
echo "https.truststore.path=keystore.jks" | sudo tee -a $XF_CONF/transformer.properties > /dev/null
echo "https.truststore.password=password" | sudo tee -a $XF_CONF/transformer.properties > /dev/null
echo "https.cluster.keystore.path=keystore.jks" | sudo tee -a $XF_CONF/transformer.properties > /dev/null
echo "https.cluster.keystore.password=password" | sudo tee -a $XF_CONF/transformer.properties > /dev/null
echo "https.cluster.truststore.path=keystore.jks" | sudo tee -a $XF_CONF/transformer.properties > /dev/null
echo "https.cluster.truststore.password=password" | sudo tee -a $XF_CONF/transformer.properties > /dev/null

TOTAL_MEM=$(free -m | grep Mem | awk '{print $2}')
JAVA_MEM=$(awk "BEGIN {printf \"%.0f\",$TOTAL_MEM*.8}")
echo -e "\tIncreasing JVM memory to ${JAVA_MEM}M"
sudo sed -i "s/-Xmx1024m -Xms1024m/-Xmx${JAVA_MEM}m -Xms${JAVA_MEM}m/" $XF_HOME/libexec/transformer-env.sh

if [ $TOTAL_MEM -gt 8192 ]
then
  echo -e "\tChanging garbage collector to G1GC"
  sudo sed -i 's/-XX:+UseConcMarkSweepGC -XX:+UseParNewGC/-XX:+UseG1GC/' $XF_HOME/libexec/transformer-env.sh
fi

echo -e "Installing Transformer CLI options"
echo "export TRANSFORMER_CLI_JAVA_OPTS=\"-Djavax.net.ssl.trustStore=$XF_CONF/keystore.jks -Djavax.net.ssl.trustStorePassword=password\"" | sudo tee -a /etc/profile.d/transformer.sh > /dev/null
echo "PATH=\$PATH:$XF_HOME/bin" | sudo tee -a /etc/profile.d/transformer.sh > /dev/null
source /etc/profile.d/transformer.sh

echo "export SPARK_HOME=$SPARK_HOME" | sudo tee -a /etc/profile.d/spark.sh > /dev/null
echo "export PATH=\$PATH:$SPARK_HOME/bin" | sudo tee -a /etc/profile.d/spark.sh > /dev/null
source /etc/profile.d/spark.sh

echo "export SPARK_HOME=$SPARK_HOME" | sudo tee -a $XF_HOME/libexec/transformer-env.sh > /dev/null

echo "Setting system timezone to UTC"
sudo timedatectl set-timezone UTC

echo "Installing transformer service files"
sudo cp $XF_HOME/systemd/transformer.service /etc/systemd/system/transformer.service
sudo cp $XF_HOME/systemd/transformer.socket /etc/systemd/system/transformer.socket
sudo systemctl daemon-reload

echo "Starting ntp service"
sudo /lib/systemd/systemd-sysv-install enable ntp
sudo systemctl -q start ntp

echo "Starting transformer service"
sudo systemctl -q start transformer
sudo systemctl -q enable transformer

until [ $($XF_HOME/bin/streamsets cli -U https://localhost:19630 ping | grep -c info) -eq 1 ]
do 
  echo -e "\tWaiting for transformer to start..."
  sleep 5 
done

echo "Done"