List EC2-Instances

ec2_instances.py

"""
Generate a list of EC2 instances with the information that I most need to see, including estimated monthly prices.

Add a line to your .bash_profile like this:

alias list='/usr/local/bin/python3 /path/to/ec2_instances.py'

This assumes that you have aws cli installed...
"""

import boto3
import argparse
import json
import datetime
import time
import tempfile
import os
import pickle
from collections import defaultdict


tmpdir = tempfile.gettempdir()

aws_region_map = {
  'ca-central-1': 'Canada (Central)',
  'ap-northeast-3': 'Asia Pacific (Osaka-Local)',
  'us-east-1': 'US East (N. Virginia)',
  'ap-northeast-2': 'Asia Pacific (Seoul)',
  'us-gov-west-1': 'AWS GovCloud (US)',
  'us-east-2': 'US East (Ohio)',
  'ap-northeast-1': 'Asia Pacific (Tokyo)',
  'ap-south-1': 'Asia Pacific (Mumbai)',
  'ap-southeast-2': 'Asia Pacific (Sydney)',
  'ap-southeast-1': 'Asia Pacific (Singapore)',
  'sa-east-1': 'South America (Sao Paulo)',
  'us-west-2': 'US West (Oregon)',
  'eu-west-1': 'EU (Ireland)',
  'eu-west-3': 'EU (Paris)',
  'eu-west-2': 'EU (London)',
  'us-west-1': 'US West (N. California)',
  'eu-central-1': 'EU (Frankfurt)'
}

ebs_name_map = {
  'standard': 'Magnetic',
  'gp2': 'General Purpose',
  'io1': 'Provisioned IOPS',
  'st1': 'Throughput Optimized HDD',
  'sc1': 'Cold HDD'
}


def build_pricing_defaults(region='us-east-1'):
    pricing_client = boto3.client('pricing', region_name=region)
    ebs_pricing = get_ebs_pricing(client=pricing_client, region=region)
    ondemand_pricing = get_ondemand_pricing(client=pricing_client, region=region)
    spot_pricing = get_spot_pricing(region=region)
    
    file_path = build_pricing_path('aws_ebs_prices', region)
    if not pricing_file_is_good(file_path):
        with open(file_path, 'wb') as f:
            pickle.dump(ebs_pricing, f)
    
    file_path = build_pricing_path('aws_spot_prices', region)
    if not pricing_file_is_good(file_path):
        with open(file_path, 'wb') as f:
            pickle.dump(spot_pricing, f)
    
    file_path = build_pricing_path('aws_ondemand_prices', region)
    if not pricing_file_is_good(file_path):
        with open(file_path, 'wb') as f:
            pickle.dump(ondemand_pricing, f)
    return


def build_pricing_path(n, region='us-east-1'):
    return os.path.abspath(os.path.join(tmpdir, '{}-{}'.format(n, region)))


def pricing_file_is_good(file_path, ttl=604800):
    return os.path.exists(file_path) and os.path.getctime(file_path) > (time.time() - 604800)


def get_existing(file_path, ttl=604800):
    if pricing_file_is_good(file_path, ttl):
        return pickle.load(open(file_path, 'rb'))


def get_ebs_pricing(client=None, region='us-east-1', *args, **kwargs):
    """
    Returns a pricing dictionary for EBS pricing for this region
    """
    price_dictionary = get_existing(build_pricing_path('aws_ebs_prices', region))
    if price_dictionary:
        return price_dictionary
    
    if not client:
        client = boto3.client('pricing', region_name=region)
    
    resolved_region = aws_region_map.get(region)
    price_dictionary = dict()
    for ebs_code in ebs_name_map:
        response = client.get_products(ServiceCode='AmazonEC2', Filters=[
            {'Type': 'TERM_MATCH', 'Field': 'volumeType', 'Value': ebs_name_map[ebs_code]}, 
            {'Type': 'TERM_MATCH', 'Field': 'location', 'Value': resolved_region}
        ])
        for result in response['PriceList']:
            json_result = json.loads(result)
            for json_result_level_1 in json_result['terms']['OnDemand'].values():
                for json_result_level_2 in json_result_level_1['priceDimensions'].values():
                    for price_value in json_result_level_2['pricePerUnit'].values():
                        continue
        price_dictionary[ebs_code] = float(price_value)
    return price_dictionary


def get_spot_pricing(client=None, region='us-east-1', instance_types=None, *args, **kwargs):
    """
    Return spot pricing for the specified instance sizes
    """
    price_dictionary = get_existing(build_pricing_path('aws_spot_prices', region))
    if price_dictionary:
        return price_dictionary
    
    if not client:
        client = boto3.client('ec2', region_name=region)
    
    price_dictionary = defaultdict(dict)
    next_token = ''
    while True:    
        if instance_types:
            response=client.describe_spot_price_history(ProductDescriptions=['Linux/UNIX (Amazon VPC)'], StartTime=datetime.datetime.now(), InstanceTypes=instance_types, NextToken=next_token)
        else:
            response=client.describe_spot_price_history(ProductDescriptions=['Linux/UNIX (Amazon VPC)'], StartTime=datetime.datetime.now(), NextToken=next_token)
        for x in response['SpotPriceHistory']:
            price_dictionary[x['InstanceType']][x['AvailabilityZone']] = float(x['SpotPrice'])
        next_token = response.get('NextToken')
        if not next_token:
            break
    
    return price_dictionary


def get_ondemand_pricing(client=None, region='us-east-1', instance_types=None, *args, **kwargs):
    price_dictionary = get_existing(build_pricing_path('aws_ondemand_prices', region))
    if price_dictionary:
        return price_dictionary
    
    if not client:
        client = boto3.client('pricing', region_name=region)
    
    resolved_region = aws_region_map.get(region)
    prices = dict()
    next_token = ''
    while True:
        response = client.get_products(
            ServiceCode='AmazonEC2',
            Filters=[
                {'Type': 'TERM_MATCH', 'Field': 'operatingSystem', 'Value': 'Linux'},
                {'Type': 'TERM_MATCH', 'Field': 'tenancy', 'Value': 'Shared'},
                {'Type': 'TERM_MATCH', 'Field': 'location', 'Value': resolved_region},
                {'Type': 'TERM_MATCH', 'Field': 'capacitystatus', 'Value': 'Used'},
                {'Type': 'TERM_MATCH', 'Field': 'preInstalledSw', 'Value': 'NA'}
            ],
            NextToken=next_token
        )
        
        for x in response['PriceList']:
            p = json.loads(x)
            instance_type = p['product']['attributes']['instanceType']
            instance_cost = 0.00
            for terms in p['terms']['OnDemand'].values():
                for price_dimensions in terms['priceDimensions'].values():
                    instance_cost = float(price_dimensions['pricePerUnit'].get('USD', 0))
                    prices[instance_type] = instance_cost
        
        next_token = response.get('NextToken')
        if not next_token:
            break
    return prices


def get_value_from_tags(tags, key):
  if not tags:
    return
  for tag in tags:
    if tag.get('Key').lower() == key.lower():
      return tag.get('Value')


def get_instance_name(i):
  return get_instance_name_from_tags(i.tags)


def get_instance_name_from_tags(tags):
  return get_value_from_tags(tags, 'name')


def get_division(i):
  return get_division_from_tags(i.tags)


def get_division_from_tags(tags):
  return get_value_from_tags(tags, 'division')


def get_owner(i):
  return get_owner_from_tags(i.tags)


def get_owner_from_tags(tags):
  return get_value_from_tags(tags, 'owner')


def get_projects(i):
    return get_projects_from_tags(i.tags)


def get_projects_from_tags(tags):
  project = get_value_from_tags(tags, 'project')
  if not project:
    project = 'unspecified'
  return [p.strip() for p in project.split(',')]


def get_start_date(i):
    if i.state.get('Name') in ('running', 'stopping', 'starting'):
        return i.launch_time.strftime('%Y-%m-%d %H:%M')
    return ''


def get_volume_size(i):
    return sum((v.size for v in i.volumes.iterator()))


def get_volume_cost(ebs_prices, volumes):
    return sum(v.size * ebs_prices[v.volume_type] for v in volumes)


def is_spot(i):
    return i.instance_lifecycle == 'spot'


def gather_instances(instances, filter=None, states=None, owners=None, projects=None, unowned=None, region='us-east-1', *args, **kwargs):
    filter = filter or ""
    filters = []
    
    if filter:
        filters.append({'Name':'tag:Name', 'Values':['*{}*'.format(filter)]})
    
    if owners:
        values = []
        for o in owners:
            values.append('*{}*'.format(o))   
        filters.append({'Name':'tag:owner', 'Values':values})
    
    if projects:
        values = []
        for p in projects:
            values.append('*{}*'.format(p))   
        filters.append({'Name':'tag:project', 'Values':values})
        
    if filters:
        instance_list = list(instances.filter(Filters=filters))
    else:
        instance_list = list(instances.all())
    
    
    instance_types = {i.instance_type for i in instance_list}
    ebs_prices = get_ebs_pricing(region=region)
    spot_prices = get_spot_pricing(instance_types=list(instance_types), region=region)
    ond_prices = get_ondemand_pricing(instance_types=list(instance_types), region=region)
    
    for i in instance_list:
        tags = i.tags
        id = i.id
        name = get_instance_name_from_tags(tags) or ''
        owner = get_owner_from_tags(tags) or ''
        division = get_division_from_tags(tags) or None
        projects = get_projects_from_tags(tags) or list()
        
        status = i.state.get('Name')
        volumes = list(i.volumes.all())
        volume_size = sum((v.size for v in volumes))
        volume_cost = get_volume_cost(ebs_prices, volumes)
        availability_zone = i.placement.get('AvailabilityZone')
        if states and status not in states:
            continue
        
        if unowned and owner:
            continue
        
        public_ip = i.public_ip_address or ''
        private_ip = i.private_ip_address or ''
        start_date = get_start_date(i)
        instance_type = i.instance_type or ''
        spot_instance = is_spot(i)
        
        instance_price = 0
        if spot_instance:
            instance_price = spot_prices[instance_type][availability_zone] * 730  # 8760 hours (in a year) / 12
        else:
            instance_price = ond_prices[instance_type] * 730  # 8760 hours (in a year) / 12
        
        instance_total = volume_cost + instance_price
        yield (id, instance_type, name.lower(), status, public_ip, private_ip, start_date, owner, spot_instance, volume_size, volume_cost, instance_price, instance_total), (owner, division, projects)


def print_instance_header(i):
    print("Instances:")
    print("–" * 203)
    print("{!s:20}  {!s:12}  {!s:40}  {!s:7}  {!s:15}  {!s:15}  {!s:17}  {!s:12}  {!s:5}  {!s:>9}  {!s:>9}  {!s:>9}  {!s:>9}".format(*i))


def print_instance(i):
    #print(i)
    print("{!s:20}  {!s:12}  {!s:40}  {!s:7}  {!s:15}  {!s:15}  {!s:17}  {!s:12}  {!s:5}  {!s:>9}  {:9.2f}  {:9.2f}  {:9.2f}".format(*i))


def print_summary_costs(title, cost_dict):
    print(title)
    print("––––––––––––––––––––––––––––––––––")
    costs = sorted(cost_dict.items(), key=lambda x: x[0])
    for a,b in costs:
        print("   {!s:20}: {:9.2f}".format(a, b))
    print()
    

def main(filter=None, states=None, region='us-east-1', owners=None, projects=None, unowned=None, *args, **kwargs):
    ec2 = boto3.resource('ec2', region)
    build_pricing_defaults(region=region)
    states = states or ('pending', 'running', 'shutting-down', 'terminated', 'stopping', 'stopped')
    owners = owners or None
    instances = sorted(gather_instances(ec2.instances, filter=filter, states=states, owners=owners, projects=projects, unowned=unowned, region=region), key=lambda x: (x[0][2], x[0][6]))
    print_instance_header(('id', 'type', 'name', 'state', 'public ip', 'private ip', 'launched', 'owner', 'spot', 'disk (gb)', 'disk $/m', '$/month', 'total'))
    project_costs = defaultdict(float)
    owner_costs = defaultdict(float)
    division_costs = defaultdict(float)
    for i, (owner, division, projects) in instances:
        print_instance(i)
        for p in projects:
            project_costs[p] += i[12]
        owner_costs[owner or 'unspecified'] += i[12]
        division_costs[division or 'unspecified'] += i[12]
    
    disk_total = sum(i[10] for i, x in instances)
    instance_total = sum(i[11] for i, x in instances)
    total_cost = sum(i[12] for i, x in instances)
    print("{:>181.2f}  {:9.2f}  {:9.2f}".format(disk_total, instance_total, total_cost))
    print()
    print_summary_costs('Monthly Cost Summary By Owner', owner_costs)
    print_summary_costs('Monthly Cost Summary By Division', division_costs)
    print_summary_costs('Monthly Cost Summary By Project', project_costs)
    return True


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='List EC2 Instances')
    parser.add_argument("filter", nargs='?', default=None, help="Filter by name or id")
    parser.add_argument("-s", "--state", dest="state", default='running', nargs="*", help="Return instances in this state")
    parser.add_argument("-r", "--region", dest="region", default="us-east-1", help="Return instances in this region")
    parser.add_argument("-o", "--owners", dest="owners", default=None, nargs="*", help="Filter based on owners")
    parser.add_argument("-p", "--projects", dest="projects", default=None, nargs="*", help="Filter based on projects")
    parser.add_argument("-u", "--unowned", dest="unowned", action='store_true', help="Return instances that do not have a declared owner")
    parser.add_argument("-t", "--tempdir", dest="temp", default=tempfile.gettempdir(), help="Temp directory to store pricing lists")
    options = parser.parse_args()
    tmpdir = options.temp
    main(filter=options.filter, states=options.state, region=options.region, owners=options.owners, projects=options.projects, unowned=options.unowned)