Convert Cattle to Pets

aws-ec2-spot-fleet-role

{
  "Version": "2012-10-17",
  "Statement": [{
    "Effect": "Allow",
    "Action": [
       "ec2:DescribeImages",
       "ec2:DescribeSubnets",
       "ec2:RequestSpotInstances",
       "ec2:TerminateInstances",
       "ec2:DescribeInstanceStatus",
       "iam:PassRole"
        ],
    "Resource": ["*"]
  }]
}

clear-network

#!/bin/bash
for f in $(ls /etc/sysconfig/network-scripts/ifcfg-* | grep -v lo);
do
    rm -f ${f}
done

clear-network.service

[Unit]
Description=Remove /etc/sysconfig/network-scripts/ifcfg-* on shutdown

[Service]
Type=oneshot
RemainAfterExit=true
ExecStop=/usr/bin/clear-network

[Install]
WantedBy=multi-user.target

instance-profile

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "Stmt1500333055000",
            "Effect": "Allow",
            "Action": [
                "ec2:AttachVolume",
                "ec2:DescribeVolumes"
            ],
            "Resource": [
                "*"
            ]
        }
    ]
}

spotter

#!/bin/bash

service_tag=spotter
role_tag=gpu_research

spot_fleet_request_validity="5min"
spot_fleet_role_name=aws-ec2-spot-fleet-role
spot_price_history_time="1day"

preboot_image_id=ami-50f9d935
instance_type=p2.xlarge
instance_type=g3.4xlarge
public_key_name=ohio-gpu

ssh_username=fedora
ssh_port=314

echo -n "Discovering Volume... "
volume_data=$(aws --profile ${service_tag}_${role_tag} \
              ec2 describe-volumes \
                  --filters Name=tag:service,Values=${service_tag} \
                            Name=tag:role,Values=${role_tag} \
                  --output=json)
volume_data=$(echo "${volume_data}" | jq .Volumes[0])
if [ "x${volume_data}" == "xnull" ]; then
    echo "Error discovering volume"
    exit 10
fi

volume_id=$(echo "${volume_data}" | jq -r .VolumeId)
if [ "x${volume_id}" == "x" ]; then
    echo "Error discovering volume id"
    exit 10
fi

volume_az=$(echo "${volume_data}" | jq -r .AvailabilityZone)
if [ "x${volume_az}" == "x" ]; then
    echo "Error discovering volume availability zone"
    exit 10
fi
echo "done"

echo -n "Discovering IAM instance Profile... "
profile_data=$(aws --profile ${service_tag}_${role_tag} \
               iam get-instance-profile \
                   --instance-profile-name "${role_tag}" \
                   --output=json)
profile_data=$(echo "${profile_data}" | jq .InstanceProfile)
if [ "x${profile_data}" == "x" ]; then
    echo "Error discovering instance profile"
    echo "Make sure there is an instance profile named ${role_tag}"
    exit 10
fi

profile_arn=$(echo "${profile_data}" | jq -r .Arn)
if [ "x${profile_arn}" == "x" ]; then
    echo "Error discovering instance profile arn"
    exit 10
fi
echo "done"

echo -n "Discovering IAM spot fleet role... "
fleet_role_data=$(aws --profile ${service_tag}_${role_tag} \
                  iam get-role \
                      --role-name "${spot_fleet_role_name}" \
                      --output=json)
fleet_role_data=$(echo "${fleet_role_data}" | jq .Role)
if [ "x${fleet_role_data}" == "x" ]; then
    echo "Error discovering spot fleet role"
    echo "Make sure there is an spot fleet role named ${spot_fleet_role_name}"
    exit 10
fi

fleet_role_arn=$(echo "${fleet_role_data}" | jq -r .Arn)
if [ "x${fleet_role_arn}" == "x" ]; then
    echo "Error discovering spot fleet role arn"
    exit 10
fi
echo "done"

echo -n "Discovering VPC... "
vpc_data=$(aws --profile ${service_tag}_${role_tag} \
           ec2 describe-vpcs \
               --filters Name=is-default,Values=true \
               --output=json)
vpc_data=$(echo "${vpc_data}" | jq .Vpcs[0])
if [ "x${vpc_data}" == "xnull" ]; then
    echo "Error discovering default vpc"
    exit 10
fi

vpc_id=$(echo "${vpc_data}" | jq -r .VpcId)
if [ "x${vpc_id}" == "x" ]; then
    echo "Error discovering vpc id"
    exit 10
fi
echo "done"

echo -n "Discovering Subnet... "
subnet_data=$(aws --profile ${service_tag}_${role_tag} \
              ec2 describe-subnets \
                  --filters Name=vpc-id,Values="${vpc_id}" \
                            Name=availability-zone,Values="${volume_az}" \
                  --output=json)
subnet_data=$(echo "${subnet_data}" | jq .Subnets[0])
if [ "x${subnet_data}" == "xnull" ]; then
    echo "Error discovering subnet"
    exit 10
fi

subnet_id=$(echo "${subnet_data}" | jq -r .SubnetId)
if [ "x${subnet_id}" == "x" ]; then
    echo "Error discovering subnet id"
    exit 10
fi
echo "done"

echo -n "Discovering Security Group... "
sg_data=$(aws --profile ${service_tag}_${role_tag} \
          ec2 describe-security-groups \
              --filters Name=tag:service,Values=${service_tag} \
                        Name=tag:role,Values=${role_tag} \
              --output=json)
sg_data=$(echo "${sg_data}" | jq .SecurityGroups[0])
if [ "x${sg_data}" == "xnull" ]; then
    echo "Error discovering security group"
    exit 10
fi

sg_id=$(echo "${sg_data}" | jq -r .GroupId)
if [ "x${sg_id}" == "x" ]; then
    echo "Error discovering security group id"
    exit 10
fi
echo "done"

echo -n "Generating Userdata... "
read -r -d '' user_data <<"EOF"
#!/bin/bash
dnf update
dnf install -y jq python-pip
pip install awscli
function swap_root_volume {
 echo "Swapping root volume"
 INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
 ZONE=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone)
 REGION=${ZONE::-1}
 aws ec2 attach-volume --volume-id ${VOLUME_ID} \
                       --instance-id ${INSTANCE_ID} \
                       --device /dev/sdf \
                       --region ${REGION} || exit -1
 while ! lsblk /dev/xvdf
 do
  echo "waiting for device to attach"
  sleep 5
 done

 # Ready up for the swap
 DEVICE=/dev/xvdf1
 NEW_UUID=$(uuidgen)
 e2fsck -f -p ${DEVICE}
 tune2fs-U ${NEW_UUID} ${DEVICE}

 EXISTING=$(df --output=source / | tail -n1)
 OLD_UUID=$(blkid -s UUID -o value ${EXISTING})

 sed -i "s/${OLD_UUID}/${NEW_UUID}/g" /boot/grub/grub.conf
 sed -i "s/${OLD_UUID}/${NEW_UUID}/g" /boot/grub2/grub.cfg
}
EOF

user_data+="
SERVICE_TAG=${service_tag}
ROLE_TAG=${role_tag}
VOLUME_ID=${volume_id}
VOLUME_AZ=${volume_az}
swap_root_volume
shutdown -r now
"

user_data=$(echo "${user_data}" | base64 | tr -d '\n')
echo "done"

echo -n "Generating Instance Spec... "
read -r -d '' instance_spec <<EOF
{
  "ImageId" : "${preboot_image_id}",
  "InstanceType": "${instance_type}",
  "KeyName" : "${public_key_name}",
  "EbsOptimized": true,
  "Placement": {
     "AvailabilityZone": "${volume_az}"
  },
  "IamInstanceProfile": {
    "Arn": "${profile_arn}"
  },
  "BlockDeviceMappings": [
    {
      "DeviceName": "/dev/sda1",
      "Ebs": {
        "DeleteOnTermination": true,
        "VolumeType": "gp2",
        "VolumeSize": 8
      }
    }
  ],
  "NetworkInterfaces": [
      {
        "DeviceIndex": 0,
        "SubnetId": "${subnet_id}",
        "Groups": [ "${sg_id}" ],
        "AssociatePublicIpAddress": true
      }
  ],
  "UserData" : "${user_data}"
}
EOF
echo "done"

echo -n "Fetching spot price history... "
spot_price_history_start=$(date -u -d "-${spot_price_history_time}" \
                                      +%Y-%m-%dT%H:%M:%SZ)
spot_price_data=$(aws --profile ${service_tag}_${role_tag} \
                  ec2 describe-spot-price-history \
                      --start-time "${spot_price_history_start}" \
                      --instance-types "${instance_type}" \
                      --filters Name=availability-zone,Values="${volume_az}" \
                                Name=product-description,Values=Linux/UNIX \
                  --output=json)
spot_price_data=$(echo "${spot_price_data}" | jq .SpotPriceHistory)
if [ "x${spot_price_data}" == "xnull" ]; then
    echo "Error collecting spot price data"
    exit 10
fi
echo "done"

if hash gnuplot 2>/dev/null; then
    IFS= gnuplot_plot_data=$(echo "${spot_price_data}" |
                             jq -r '.[] | "\(.Timestamp) \(.SpotPrice)"')

    gnuplot <<EOF
set terminal dumb

\$spot_price << EOD
$gnuplot_plot_data
EOD
set autoscale fix
set title "Spot Bid Price History"
set ylabel "Price"
set xlabel "Time"
set xdata time
set timefmt "%Y-%m-%dT%H:%M:%S.000Z"
set grid
set nokey
plot \$spot_price using 1:2 pt "*"
EOF
else
    echo ''
    echo "gnuplot not found in PATH, consider installing for price graphing."
    echo "Only showing last 10 prices"
    echo ''

    echo "${spot_price_data}" |
         jq -r '.[0:10][] | "\(.Timestamp)\t\(.SpotPrice)"'
fi

spot_prices=$(echo "${spot_price_data}" |
              jq '[ .[] | .SpotPrice | tonumber ]')



echo "Min Price: $(echo "${spot_prices}" | jq 'min')"
echo "Max Price: $(echo "${spot_prices}" | jq 'max')"
echo "Average Price: $(echo "${spot_prices}" | jq 'add/length')"
echo -n "Standard Deviation: "
echo "${spot_prices}" | jq '(map(.*.)|add/length)-pow(add/length;2)|sqrt'
echo ''

read -r -p "Spot bid price: " spot_price

echo -n "Generating Spot Fleet Request... "
fleet_req_valid_until=$(date -u -d "+${spot_fleet_request_validity}" \
                                   +%Y-%m-%dT%H:%M:%SZ)
read -r -d '' spot_fleet_req <<EOF
{
  "IamFleetRole": "${fleet_role_arn}",
  "AllocationStrategy": "lowestPrice",
  "TargetCapacity": 1,
  "SpotPrice": "${spot_price}",
  "ValidUntil": "${fleet_req_valid_until}",
  "TerminateInstancesWithExpiration": false,
  "LaunchSpecifications": [ ${instance_spec} ],
  "Type": "request"
}
EOF
echo "done"

echo -n "Requsting Spot Fleet... "
fleet_req_data=$(aws --profile ${service_tag}_${role_tag} \
                ec2 request-spot-fleet \
                    --spot-fleet-request-config "${spot_fleet_req}" \
                    --output=json)

fleet_req_id=$(echo "${fleet_req_data}" | jq -r .SpotFleetRequestId)
if [ "x${fleet_req_id}" == "x" ]; then
    echo "Error requesting spot fleet"
    echo "${fleet_req_data}"
    exit 10
fi
echo "done"

echo ''
echo "Spot Fleet Request ID: ${fleet_req_id}"
echo ''

echo -n "Waiting for instance to come online... "
while true; do
    fleet_data=$(aws --profile ${service_tag}_${role_tag} \
                 ec2 describe-spot-fleet-instances \
                     --spot-fleet-request-id "${fleet_req_id}" \
                     --output=json)
    fleet_data=$(echo "${fleet_data}" | jq .ActiveInstances[0])
    if [ "x${fleet_data}" == "xnull" ]; then
        sleep 5
        continue
    fi

    instance_id=$(echo "${fleet_data}" | jq -r .InstanceId)
    if [ "x${instance_id}" == "x" ]; then
        sleep 5
        continue
    else
        break
    fi
done
echo "done"

echo ''
echo "Instance ID: ${instance_id}"
echo ''


echo -n "Discoverying Public IP Address... "
instance_data=$(aws --profile ${service_tag}_${role_tag} \
                ec2 describe-instances \
                    --instance-ids "${instance_id}" \
                    --output=json)

instance_data=$(echo "${instance_data}" | jq .Reservations[0])
instance_data=$(echo "${instance_data}" | jq .Instances[0])
if [ "x${instance_data}" == "xnull" ]; then
    echo "Error discovering public ip address"
    echo "${instance_data}"
    exit 10
fi

public_ip=$(echo "${instance_data}" | jq -r .PublicIpAddress)
if [ "x${public_ip}" == "x" ]; then
    echo "Error getting public ip address"
    exit 10
fi
echo "done"

echo ''
echo "Public IP: ${public_ip}"
echo ''

read -r -p "ssh to instance? [Y/n] " response
case "$response" in
    [nN][oO]|[nN])
		exit 0
        ;;
    *)
        ssh -i ~/.aws/${public_key_name}.pem \
            -o "ConnectTimeout=5" \
            -q \
			-p ${ssh_port} \
			${ssh_username}@"${public_ip}"
        ssh_exit=$?

        while [ $ssh_exit -ne 0 ] || [ $ssh_exit -ne 130 ]
        do
            ssh -i ~/.aws/${public_key_name}.pem \
                -o "ConnectTimeout=5" \
                -q \
			    -p ${ssh_port} \
			    ${ssh_username}@"${public_ip}"
            ssh_exit=$?
        done
        ;;
esac


read -r -p "destroy instance? [Y/n] " response
case "$response" in
    [nN][oO]|[nN])
		exit 0
        ;;
    *)
		aws --profile ${service_tag}_${role_tag} \
        ec2 terminate-instances \
        	--instance-ids "${instance_id}"
        ;;
esac