Skip to content

Instantly share code, notes, and snippets.

@shokoe
Last active September 28, 2021 12:17
Show Gist options
  • Save shokoe/167fc3235a449dccab1b5a6c637fee46 to your computer and use it in GitHub Desktop.
Save shokoe/167fc3235a449dccab1b5a6c637fee46 to your computer and use it in GitHub Desktop.
Export aws billing info to grafana. Uses AWS billing info from S3. Just activate logs in billing and change the 'f' var in the script.
#!/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
netcat=$(which nc)
host='<carbon IP>'
port='2003'
nc_cmd="$netcat -q0 $host $port"
get_month(){
f="<your id>-aws-billing-detailed-line-items-with-resources-and-tags-${1}"
cd /tmp
aws s3 cp s3://<billing bucket>/${f}.csv.zip /tmp/
unzip /tmp/${f}.csv.zip
cd -
cat /tmp/${f}.csv |\
sed 's#""#"EMPTY"#g; s#","#^#g; s#^"##; s#"$##;' |\
gawk -F ^ '
BEGIN{
# the PT (product tag) hash defines one user tag per product to limit the main multi-dimention array
PT["Amazon Elastic Compute Cloud"] = "user:type"
PT["Amazon Simple Storage Service"] = "user:buckMain"
# examples for adding products without a dedicated tag
PT["Amazon CloudFront"] = "zamburama"
#PT["Amazon CloudFront"] = ""
}
NR==1 {
# this is for using column headers instead of numbers
for (i=1; i<=NF; i++) {
f[$i] = i
}
}
NR!=1{
# default tag
TAG="user:type"
if ( PT[$f["ProductName"]] in f ) TAG=PT[$f["ProductName"]]
# this if limits metrics to PT array
if ( PT[$f["ProductName"]] ){
# sort input a bit more
gsub(/ /, "_")
gsub(/[()]/, "")
# UsageType include instance type wich has periods
gsub(/\./, "-", $f["UsageType"])
# get timestamp
gsub(/[^0-9]/, " ", $f["UsageStartDate"])
D=mktime($f["UsageStartDate"])
# main monster hash
clus[D][$f["ProductName"]][$f["UsageType"]][$f["Operation"]][$f[TAG]]+=$f["Cost"]
}
}
END{
for (d in clus){
for (p in clus[d]){
for (u in clus[d][p]){
for (o in clus[d][p][u]){
for (t in clus[d][p][u][o]){
printf "aws_billing.%s %.2f %d\n", p"."u"."o"."t, clus[d][p][u][o][t], d
# extract and report instances specific data
if ( u ~ /BoxUsage:/ || u ~ /^HeavyUsage:/ || u ~ /^SpotUsage:/ ){
# extract instance types
split(u,z,":")
# and family
split(z[2],y,"-")
# report omdemand
if ( u ~ /BoxUsage:/ || u ~ /^HeavyUsage:/ ){
printf "aws_billing_instances.%s %.2f %d\n", p"."u"."o"."y[1]"."z[2]".ondemand."t, clus[d][p][u][o][t], d
}
# report spot
if ( u ~ /^SpotUsage:/ ){
split(u,z,":")
printf "aws_billing_instances.%s %.2f %d\n", p"."u"."o"."y[1]"."z[2]".spot."t, clus[d][p][u][o][t], d
}
}
}
}
}
}
}
}
' | $nc_cmd
rm -f /tmp/${f}.csv &>/dev/null
}
get_month `date -d "-1 hour" +%Y-%m`
exit
#for i in `seq 14`; do
# D=`date -d "-$i month" +%Y-%m`
# echo $D
# get_month $D | $nc_cmd
#done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment