Create a gist now

Instantly share code, notes, and snippets.

@i0rek /spotpricewatcher.sh Secret
Last active Oct 9, 2017

What would you like to do?
Tools to monitor spot instances, spot requests and spot prices. We are using local statsds to feed either librato or a system we call autobots.
#!/bin/bash
instance_types=$(aws ec2 describe-spot-instance-requests | jq -r '.SpotInstanceRequests[].LaunchSpecification.InstanceType'| sort -u | xargs)
start_time=$(TZ=CST+24 date "+%Y-%m-%dT%H:%M:%S.000Z")
statsd_lines=$(aws ec2 describe-spot-price-history --instance-types $instance_types --start-time $start_time --product-descriptions "Linux/UNIX" --max-items 1000 | jq -r '.SpotPriceHistory | sort_by(.Timestamp) | reverse | group_by(.InstanceType, .AvailabilityZone) | .[][0] | "watcher.spotprice." +.InstanceType + "." + .AvailabilityZone + ":" + .SpotPrice + "|g"')
for l in $statsd_lines; do
echo $l
echo "$l" | nc -w 1 -u localhost 8125
echo "$l" | nc -w 1 -u localhost 8126
done
#!/bin/bash
statsd_lines=$(aws ec2 describe-spot-instance-requests | jq -c -r '.SpotInstanceRequests[] | .LaunchSpecification.ImageId + "." + .InstanceId + " " + .Status.Code' | awk '/marked-for-termination|instance-terminated-by-price|instance-terminated-no-capacity|instance-terminated-capacity-oversubscribed/ {print "watcher.spotrequest.termination." $1 ":1|c"}')
for l in $statsd_lines; do
echo $l
echo "$l" | nc -w 1 -u localhost 8126
done
package main
import (
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"regexp"
"time"
)
var librato_statsd net.Conn
var autobot_statsd net.Conn
var metadataUrl = "http://169.254.169.254/latest/meta-data/"
var awsTimePattern = "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z$"
var client = &http.Client{Timeout: time.Millisecond * 500}
var debug = false
func init() {
librato_statsd, _ = net.Dial("udp", "localhost:8125")
autobot_statsd, _ = net.Dial("udp", "localhost:8126")
if os.Getenv("DEBUG") != "" {
debug = true
}
}
func report() {
if librato_statsd != nil {
librato_statsd.Write([]byte("watcher.spotinstance.termination:1|c\n"))
}
if autobot_statsd != nil {
var amiId, instanceId []byte
resp, err := client.Get(metadataUrl + "ami-id")
if err == nil && resp.StatusCode == http.StatusOK {
amiId, _ = ioutil.ReadAll(resp.Body)
}
resp, err = client.Get(metadataUrl + "instance-id")
if err == nil && resp.StatusCode == http.StatusOK {
instanceId, _ = ioutil.ReadAll(resp.Body)
}
if len(amiId) != 0 && len(instanceId) != 0 {
key := "watcher.spotinstance.termination." + string(amiId) + "." + string(instanceId) + " :1|c\n"
autobot_statsd.Write([]byte(key))
}
}
}
func aboutToTerminate() bool {
// following http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html#using-spot-instances-managing-interruptions
// > If your Spot Instance is marked for termination by the Spot service,
// > the termination-time item is present and it specifies the approximate
// > time in UTC
//
// > If the Spot service is not preparing to terminate the instance,
// > or if you terminated the Spot Instance yourself, the termination-time
// > item is either not present (so you receive an HTTP 404 error) or
// > contains a value that is not a time value.
resp, err := client.Get(metadataUrl + "spot/termination-time")
if err == nil && resp.StatusCode == http.StatusOK {
ttime, err := ioutil.ReadAll(resp.Body)
matched, err := regexp.MatchString(awsTimePattern, string(ttime))
resp.Body.Close()
if err != nil && matched == true {
return true
}
}
return false
}
func main() {
for {
if aboutToTerminate() {
if debug {
fmt.Println("about to terminate")
}
report()
} else {
if debug {
fmt.Println("don't worry")
}
}
// > we recommend every 5 seconds
time.Sleep(5 * time.Second)
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment