Skip to content

Instantly share code, notes, and snippets.

@Dextaa
Last active July 23, 2018 08:19
Show Gist options
  • Save Dextaa/3f4461733c49d80ec7cc96a8f26aa317 to your computer and use it in GitHub Desktop.
Save Dextaa/3f4461733c49d80ec7cc96a8f26aa317 to your computer and use it in GitHub Desktop.
#!/bin/bash
# Use whichever region you're working in
export AWS_DEFAULT_REGION=eu-west-1
# Make ec2-user's home dir the install to make it easier to navigate when subsequently ssh-ing in to instances
su ec2-user -
cd /home/ec2-user
echo Working dir is `pwd`
# Use the instance metadata to find out about the network the instance has been lauched into (the IP is the standard AWS metadata IP)
AWS_METADATA_IP=169.254.169.254
MAC_ADDRESS=`curl http://$AWS_METADATA_IP/latest/meta-data/network/interfaces/macs/`
SUBNET_ID=`curl http://$AWS_METADATA_IP/latest/meta-data/network/interfaces/macs/${MAC_ADDRESS}subnet-id`
SELF_INSTANCE_ID=`curl http://$AWS_METADATA_IP/latest/meta-data/instance-id`
S3_CONFIG_BUCKET=com.example.event-servers
ENI_NAME=event-servers-eni
ENI_INDEX=1
ZK_VERSION=zookeeper-3.4.9
ZK_CONFIG_FILE=zookeeper.config
KAFKA_VERSION=kafka_2.11-0.10.0.1
KAFKA_CONFIG_FILE=kafka.config
attach_eni() {
# Use the AWS CLI to see if the target ENI is already attached..
CURRENT_ATTACHMENT_ID=`aws ec2 describe-network-interfaces --filters "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --query "NetworkInterfaces[0].[Attachment][0].[AttachmentId]" | grep -o 'eni-attach-[a-z0-9]*'`
# ..and if it is, remove the attachment (the assumption here is that it's still attached to another instance that is being terminated).
if [ $CURRENT_ATTACHMENT_ID"X" != "X" ]; then
echo Detaching $CURRENT_ATTACHMENT_ID;
aws ec2 detach-network-interface --attachment-id $CURRENT_ATTACHMENT_ID
fi
# Use the AWS CLI to get the id of the ENI to be attached
NETWORK_INTERFACE_ID=`aws ec2 describe-network-interfaces --filters "Name=status,Values=available" "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --output json --query "NetworkInterfaces[0].NetworkInterfaceId" | grep -o 'eni-[a-z0-9]*'`
# Attach the ENI (and display the attachment id)
echo Attaching $ENI_NAME to $ENI_INDEX `aws ec2 attach-network-interface --network-interface-id $NETWORK_INTERFACE_ID --instance-id $SELF_INSTANCE_ID --device-index $ENI_INDEX`
}
install_zookeeper() {
# Copy the ZK config file from S3
aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$ZK_CONFIG_FILE $ZK_CONFIG_FILE
# Format one of the EBS volumes associated with the instance and mount it for use as the ZK data directory.
mkfs -t ext4 /dev/sdb
DATA_DIR=`cat $ZK_CONFIG_FILE | grep 'dataDir=[a-z0-9/-]*' | sed 's/dataDir=//'`
# This assumes that the dataDir value in the ZK config has a parent directory. The volume is mounted at the parent rather than the 'dataDir' value because the
# ext4 file system adds a 'lost+found' directory at the mount point and this confuses ZK.
DATA_PARENT=`dirname "${DATA_DIR}"`
mkdir -p $DATA_PARENT
mount /dev/sdb $DATA_PARENT
mkdir -p $DATA_DIR
chmod a+rwx $DATA_DIR
# As per the dataDir
mkfs -t ext4 /dev/sdc
LOG_DIR=`cat $ZK_CONFIG_FILE | grep 'dataLogDir=[a-z0-9/-]*' | sed 's/dataLogDir=//'`
LOG_PARENT=`dirname "${LOG_DIR}"`
mkdir -p $LOG_PARENT
mount /dev/sdc $LOG_PARENT
mkdir -p $LOG_DIR
chmod a+rwx $LOG_DIR
# Find the 'server.<number>' entry in the ZK config whose IP address matches the ENI's address, extract the <number> part and use it as the ZK server's id
ENI_PRIVATE_IP=`aws ec2 describe-network-interfaces --filters "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --output json --query "NetworkInterfaces[0].PrivateIpAddresses[0].PrivateIpAddress" | grep -o '[0-9\.]*'`
MATCHING_SERVER_ID=`cat $ZK_CONFIG_FILE | sed -n "s/^server.\([0-9]*\)=$ENI_PRIVATE_IP[\:0-9]*/\1/p"`
echo $MATCHING_SERVER_ID >> $DATA_DIR/myid
# Use the ENI's IP address as ZK's client connection listen address
echo >> $ZK_CONFIG_FILE
echo "clientPortAddress=$ENI_PRIVATE_IP" >> $ZK_CONFIG_FILE
# Get the ZK zip and install it
ZK_INSTALL_FILE=$ZK_VERSION.tar.gz
aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$ZK_INSTALL_FILE $ZK_INSTALL_FILE
tar -xzf $ZK_INSTALL_FILE
rm $ZK_INSTALL_FILE
chown -R ec2-user:ec2-user $DATA_DIR
chown -R ec2-user:ec2-user $LOG_DIR
}
start_zookeeper() {
# Optional
echo "export JMXDISABLE=true" >> /home/ec2-user/.bash_profile
echo "export SERVER_JVMFLAGS=-DXmx2G" >> /home/ec2-user/.bash_profile
# Schedule starting the ZK server to ensure that the instance is up and all configuration has run
# The time based scheduling could obviously be made more sophisticated should it be required.
runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$ZK_VERSION/bin/zkServer.sh start /home/ec2-user/$ZK_CONFIG_FILE &' | at now + 3 minute"
}
install_kafka() {
# See comments in the 'install_zookeeper' function
aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$KAFKA_CONFIG_FILE $KAFKA_CONFIG_FILE
mkfs -t ext4 /dev/sdd
LOG_DIR=`cat $KAFKA_CONFIG_FILE | grep 'log\.dir=[a-z0-9/-]*' | sed 's/log\.dir=//'`
LOG_PARENT=`dirname "${LOG_DIR}"`
mkdir -p $LOG_PARENT
mount /dev/sdd $LOG_PARENT
mkdir -p $LOG_DIR
chmod a+rwx $LOG_DIR
ENI_PRIVATE_IP=`aws ec2 describe-network-interfaces --filters "Name=tag:Name,Values=$ENI_NAME" "Name=subnet-id,Values=$SUBNET_ID" --output json --query "NetworkInterfaces[0].PrivateIpAddresses[0].PrivateIpAddress" | grep -o '[0-9\.]*'`
# Use the last octet of the ENI's IP address as the Kafka broker's id (21, 37 or 53 are used in the associated blog post)
BROKER_ID=`echo $ENI_PRIVATE_IP | awk --field-separator \. '{ print $4 }'`
echo broker.id=$BROKER_ID >> $KAFKA_CONFIG_FILE
echo >> $KAFKA_CONFIG_FILE
# Use the ENI's IP address as the Kafka listener and advertised listener values
echo listeners=PLAINTEXT://${ENI_PRIVATE_IP}:9095 >> $KAFKA_CONFIG_FILE
echo >> $KAFKA_CONFIG_FILE
echo advertised.listeners=PLAINTEXT://${ENI_PRIVATE_IP}:9095 >> $KAFKA_CONFIG_FILE
echo >> $KAFKA_CONFIG_FILE
# Get the Kafka zip and install it
KAFKA_INSTALL_FILE=$KAFKA_VERSION.tgz
aws s3 cp --region $AWS_DEFAULT_REGION s3://$S3_CONFIG_BUCKET/$KAFKA_INSTALL_FILE $KAFKA_INSTALL_FILE
tar -xzf $KAFKA_INSTALL_FILE
rm $KAFKA_INSTALL_FILE
chown -R ec2-user:ec2-user $LOG_DIR
PRIMARY_IP=`curl http://$AWS_METADATA_IP/latest/meta-data/local-ipv4`
HOST_NAME=`curl http://$AWS_METADATA_IP/latest/meta-data/local-hostname`
SHORT_HOST_NAME=`echo $HOST_NAME | awk --field-separator \. '{ print $1 }'`
# The default host name for an EC2 instance causes Kafka to throw an exception so create an appropriate hosts file entry which Kafka can use
echo >> etc/hosts
echo $PRIMARY_IP $SHORT_HOST_NAME $HOST_NAME >> /etc/hosts
}
start_kafka() {
# Schedule starting the ZK server to ensure that the instance is up and all configuration has run.
# The time based scheduling could obviously be made more sophisticated should it be required.
runuser -l ec2-user -c "echo 'nohup /home/ec2-user/$KAFKA_VERSION/bin/kafka-server-start.sh /home/ec2-user/$KAFKA_CONFIG_FILE &' | at now + 4 minute"
}
attach_eni
install_zookeeper
install_kafka
chown -R ec2-user:ec2-user .
start_zookeeper
start_kafka
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment