Tools to monitor spot instances, spot requests and spot prices. We are using local statsds to feed either librato or a system we call autobots.

spotpricewatcher.sh

#!/bin/bash

instance_types=$(aws ec2 describe-spot-instance-requests | jq -r '.SpotInstanceRequests[].LaunchSpecification.InstanceType'| sort -u | xargs)
start_time=$(TZ=CST+24 date "+%Y-%m-%dT%H:%M:%S.000Z")
statsd_lines=$(aws ec2 describe-spot-price-history --instance-types $instance_types --start-time $start_time --product-descriptions "Linux/UNIX" --max-items 1000 | jq -r '.SpotPriceHistory | sort_by(.Timestamp) | reverse |  group_by(.InstanceType, .AvailabilityZone) | .[][0] | "watcher.spotprice." +.InstanceType + "." + .AvailabilityZone + ":" + .SpotPrice + "|g"')

for l in $statsd_lines; do
  echo $l
  echo "$l" | nc -w 1 -u localhost 8125
  echo "$l" | nc -w 1 -u localhost 8126
done

spotrequestwatcher.sh

#!/bin/bash

statsd_lines=$(aws ec2 describe-spot-instance-requests | jq -c -r '.SpotInstanceRequests[] | .LaunchSpecification.ImageId + "." + .InstanceId + " " + .Status.Code' | awk '/marked-for-termination|instance-terminated-by-price|instance-terminated-no-capacity|instance-terminated-capacity-oversubscribed/ {print "watcher.spotrequest.termination." $1 ":1|c"}')

for l in $statsd_lines; do
  echo $l
  echo "$l" | nc -w 1 -u localhost 8126
done

spotwatcher.go

package main

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"regexp"
	"time"
)

var librato_statsd net.Conn
var autobot_statsd net.Conn
var metadataUrl = "http://169.254.169.254/latest/meta-data/"
var awsTimePattern = "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$"
var client = &http.Client{Timeout: time.Millisecond * 500}
var debug = false

func init() {
	librato_statsd, _ = net.Dial("udp", "localhost:8125")
	autobot_statsd, _ = net.Dial("udp", "localhost:8126")
	if os.Getenv("DEBUG") != "" {
		debug = true
	}
}

func report() {
	if librato_statsd != nil {
		librato_statsd.Write([]byte("watcher.spotinstance.termination:1|c\n"))
	}
	if autobot_statsd != nil {
		var amiId, instanceId []byte
		resp, err := client.Get(metadataUrl + "ami-id")
		if err == nil && resp.StatusCode == http.StatusOK {
			amiId, _ = ioutil.ReadAll(resp.Body)
		}
		resp, err = client.Get(metadataUrl + "instance-id")
		if err == nil && resp.StatusCode == http.StatusOK {
			instanceId, _ = ioutil.ReadAll(resp.Body)
		}
		if len(amiId) != 0 && len(instanceId) != 0 {
			key := "watcher.spotinstance.termination." + string(amiId) + "." + string(instanceId) + " :1|c\n"
			autobot_statsd.Write([]byte(key))
		}
	}
}

func aboutToTerminate() bool {
	// following http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html#using-spot-instances-managing-interruptions
	// > If your Spot Instance is marked for termination by the Spot service,
	// > the termination-time item is present and it specifies the approximate
	// > time in UTC
	//
	// > If the Spot service is not preparing to terminate the instance,
	// > or if you terminated the Spot Instance yourself, the termination-time
	// > item is either not present (so you receive an HTTP 404 error) or
	// > contains a value that is not a time value.

	resp, err := client.Get(metadataUrl + "spot/termination-time")
	if err == nil && resp.StatusCode == http.StatusOK {
		ttime, err := ioutil.ReadAll(resp.Body)
		matched, err := regexp.MatchString(awsTimePattern, string(ttime))
		resp.Body.Close()
		if err != nil && matched == true {
			return true
		}
	}
	return false
}

func main() {
	for {
		if aboutToTerminate() {
			if debug {
				fmt.Println("about to terminate")
			}
			report()
		} else {
			if debug {
				fmt.Println("don't worry")
			}
		}
		// > we recommend every 5 seconds
		time.Sleep(5 * time.Second)
	}
}