Skip to content

Instantly share code, notes, and snippets.

@gregtaylor99
Forked from shokoe/aws_billing_grafana.sh
Created March 23, 2018 19:20
Show Gist options
  • Save gregtaylor99/f44f11a0d8455ee1e7cf1d6491788c96 to your computer and use it in GitHub Desktop.
Save gregtaylor99/f44f11a0d8455ee1e7cf1d6491788c96 to your computer and use it in GitHub Desktop.
Export aws billing info to grafana. Uses AWS billing info from S3. Just activate logs in billing and change the 'f' var in the script.
#!/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
netcat=$(which nc)
host='<carbon IP>'
port='2003'
nc_cmd="$netcat -q0 $host $port"
get_month(){
f="<your id>-aws-billing-detailed-line-items-with-resources-and-tags-${1}"
cd /tmp
aws s3 cp s3://<billing bucket>/${f}.csv.zip /tmp/
unzip /tmp/${f}.csv.zip
cd -
cat /tmp/${f}.csv |\
sed 's#""#"EMPTY"#g; s#","#^#g; s#^"##; s#"$##;' |\
gawk -F ^ '
BEGIN{
# the PT (product tag) hash defines one user tag per product to limit the main multi-dimention array
PT["Amazon Elastic Compute Cloud"] = "user:type"
PT["Amazon Simple Storage Service"] = "user:buckMain"
# examples for adding products without a dedicated tag
PT["Amazon CloudFront"] = "zamburama"
#PT["Amazon CloudFront"] = ""
}
NR==1 {
# this is for using column headers instead of numbers
for (i=1; i<=NF; i++) {
f[$i] = i
}
}
NR!=1{
# default tag
TAG="user:type"
if ( PT[$f["ProductName"]] in f ) TAG=PT[$f["ProductName"]]
# this if limits metrics to PT array
if ( PT[$f["ProductName"]] ){
# sort input a bit more
gsub(/ /, "_")
gsub(/[()]/, "")
# UsageType include instance type wich has periods
gsub(/\./, "-", $f["UsageType"])
# get timestamp
gsub(/[^0-9]/, " ", $f["UsageStartDate"])
D=mktime($f["UsageStartDate"])
# main monster hash
clus[D][$f["ProductName"]][$f["UsageType"]][$f["Operation"]][$f[TAG]]+=$f["Cost"]
}
}
END{
for (d in clus){
for (p in clus[d]){
for (u in clus[d][p]){
for (o in clus[d][p][u]){
for (t in clus[d][p][u][o]){
printf "aws_billing.%s %.2f %d\n", p"."u"."o"."t, clus[d][p][u][o][t], d
# extract and report instances specific data
if ( u ~ /BoxUsage:/ || u ~ /^HeavyUsage:/ || u ~ /^SpotUsage:/ ){
# extract instance types
split(u,z,":")
# and family
split(z[2],y,"-")
# report omdemand
if ( u ~ /BoxUsage:/ || u ~ /^HeavyUsage:/ ){
printf "aws_billing_instances.%s %.2f %d\n", p"."u"."o"."y[1]"."z[2]".ondemand."t, clus[d][p][u][o][t], d
}
# report spot
if ( u ~ /^SpotUsage:/ ){
split(u,z,":")
printf "aws_billing_instances.%s %.2f %d\n", p"."u"."o"."y[1]"."z[2]".spot."t, clus[d][p][u][o][t], d
}
}
}
}
}
}
}
}
' | $nc_cmd
rm -f /tmp/${f}.csv &>/dev/null
}
get_month `date -d "-1 hour" +%Y-%m`
exit
#for i in `seq 14`; do
# D=`date -d "-$i month" +%Y-%m`
# echo $D
# get_month $D | $nc_cmd
#done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment