Create a gist now

Instantly share code, notes, and snippets.

@i0rek /spotpricewatcher.sh Secret
Last active Feb 2, 2017

Tools to monitor spot instances, spot requests and spot prices. We are using local statsds to feed either librato or a system we call autobots.
#!/bin/bash
instance_types=$(aws ec2 describe-spot-instance-requests | jq -r '.SpotInstanceRequests[].LaunchSpecification.InstanceType'| sort -u | xargs)
start_time=$(TZ=CST+24 date "+%Y-%m-%dT%H:%M:%S.000Z")
statsd_lines=$(aws ec2 describe-spot-price-history --instance-types $instance_types --start-time $start_time --product-descriptions "Linux/UNIX" --max-items 1000 | jq -r '.SpotPriceHistory | sort_by(.Timestamp) | reverse | group_by(.InstanceType, .AvailabilityZone) | .[][0] | "watcher.spotprice." +.InstanceType + "." + .AvailabilityZone + ":" + .SpotPrice + "|g"')
for l in $statsd_lines; do
echo $l
echo "$l" | nc -w 1 -u localhost 8125
echo "$l" | nc -w 1 -u localhost 8126
done
#!/bin/bash
statsd_lines=$(aws ec2 describe-spot-instance-requests | jq -c -r '.SpotInstanceRequests[] | .LaunchSpecification.ImageId + "." + .InstanceId + " " + .Status.Code' | awk '/marked-for-termination|instance-terminated-by-price|instance-terminated-no-capacity|instance-terminated-capacity-oversubscribed/ {print "watcher.spotrequest.termination." $1 ":1|c"}')
for l in $statsd_lines; do
echo $l
echo "$l" | nc -w 1 -u localhost 8126
done
package main
import (
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"regexp"
"time"
)
var librato_statsd net.Conn
var autobot_statsd net.Conn
var metadataUrl = "http://169.254.169.254/latest/meta-data/"
var awsTimePattern = "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$"
var client = &http.Client{Timeout: time.Millisecond * 500}
var debug = false
func init() {
librato_statsd, _ = net.Dial("udp", "localhost:8125")
autobot_statsd, _ = net.Dial("udp", "localhost:8126")
if os.Getenv("DEBUG") != "" {
debug = true
}
}
func report() {
if librato_statsd != nil {
librato_statsd.Write([]byte("watcher.spotinstance.termination:1|c\n"))
}
if autobot_statsd != nil {
var amiId, instanceId []byte
resp, err := client.Get(metadataUrl + "ami-id")
if err == nil && resp.StatusCode == http.StatusOK {
amiId, _ = ioutil.ReadAll(resp.Body)
}
resp, err = client.Get(metadataUrl + "instance-id")
if err == nil && resp.StatusCode == http.StatusOK {
instanceId, _ = ioutil.ReadAll(resp.Body)
}
if len(amiId) != 0 && len(instanceId) != 0 {
key := "watcher.spotinstance.termination." + string(amiId) + "." + string(instanceId) + " :1|c\n"
autobot_statsd.Write([]byte(key))
}
}
}
func aboutToTerminate() bool {
// following http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html#using-spot-instances-managing-interruptions
// > If your Spot Instance is marked for termination by the Spot service,
// > the termination-time item is present and it specifies the approximate
// > time in UTC
//
// > If the Spot service is not preparing to terminate the instance,
// > or if you terminated the Spot Instance yourself, the termination-time
// > item is either not present (so you receive an HTTP 404 error) or
// > contains a value that is not a time value.
resp, err := client.Get(metadataUrl + "spot/termination-time")
if err == nil && resp.StatusCode == http.StatusOK {
ttime, err := ioutil.ReadAll(resp.Body)
matched, err := regexp.MatchString(awsTimePattern, string(ttime))
resp.Body.Close()
if err != nil && matched == true {
return true
}
}
return false
}
func main() {
for {
if aboutToTerminate() {
if debug {
fmt.Println("about to terminate")
}
report()
} else {
if debug {
fmt.Println("don't worry")
}
}
// > we recommend every 5 seconds
time.Sleep(5 * time.Second)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment