Python3.7 Script Which Queries CloudWatch For Desired Metrics Across All Accounts And Regions And Pushes To ElasticSearch

CloudWatch-To-Elasticsearch-Metrics-Ingester.py

##
##  CloudWatch-To-Elasticsearch-Metrics-Ingester.py Script written by Jamison Roberts

##
##  Description:  This script is written in Python 3 and uses the AWS boto3 python library to make bulk calls to AWS S3 and CloudWatch services.
##                A CSV of all Federated AWS accounts and names is downloaded from S3 and then the script performs a for loop on each account.
##                Within each account the script will perform a for loop on each region specified, create a cloudwatch boto3 client, and then
##                query CloudWatch Metrics to pull metrics counts for each metric provided in the call.  Then the total returned metrics count
##                for each account are totalled and printed on the console. 


##  Import required libraries used in this script
from elasticsearch import Elasticsearch, RequestsHttpConnection
from elasticsearch import helpers
from requests_aws4auth import AWS4Auth
import boto3
import json
from botocore.client import Config
from datetime import datetime, timedelta, date
import time
import random
import string


## Create a variable called ScriptStartTime with the current date and time which we will use to calculate the total script runtime with
ScriptStartTime = datetime.now()


##  Initial variables
RegionsArray = ['us-east-1','eu-west-1','us-west-1']
region = 'us-east-1'
thisaccount = '<AWS_ACCOUNT_NUMBER_THIS_IS_RUNNING_IN>'
RoleArn = 'arn:aws:iam::' + thisaccount + ':role/<ROLE_NAME_TO_ASSUME>'
docs = []
previoustimestamp = ""

##  Create our AWS S3 boto3 resource (different than the S3 boto3 client)
##  Download the csv of all AWS Federated Accounts and then read it into the variable accounts
##  Convert the accounts variable from json into a list
s3resource = boto3.resource('s3', config=Config(signature_version='s3v4'), region_name="us-east-1")
randomstring = ''.join(random.choices(string.ascii_uppercase + string.digits, k = 10))
s3resource.meta.client.download_file('<S3_Bucket>', '<S3_OBJECT_PATH>/AccountNumbersAndNames.csv', 'C:\\Tempy\\'+ randomstring + 'accounts.csv')
f = open('C:\\Tempy\\'+ randomstring + 'accounts.csv', 'r')

accounts = f.read()
accounts = json.loads(accounts)


# COMMENT THE BELOW LINE OUT!!!!!  Used For Testing so the script will only run against 1 account rather than all accounts
#accounts = ['<AWS_ACCOUNT_NUMBER>:<AWS_ACCOUNT_ALIAS>']


##  Creates the AWS STS boto3 client which we call later in the script to obtain temp credentials for each account
stsClient = boto3.client('sts')
fifteenMinutesAgo = datetime.utcnow() - timedelta(hours=0, minutes=15)
fourtyFiveMinutesAgo = datetime.utcnow() - timedelta(hours=0, minutes=45)

##  Set our initial policy invocation counters to 0 and then create the headers row for the csvFileData
##<FOREACH METRIC-LABEL Total<METRIC-LABEL>Values = 0>
TotaloldaccesskeysdisableValues = 0
TotalappelbupdateoldsslValues = 0
TotalunencrypteddynamodbtabledeletedValues = 0
Totalunencryptedec2terminatedbypolicyValues = 0


## Connect to Elasticsearch Service in AWS
host = '<AWS_ELASTICSEARCH_ARN>'
stsClient = boto3.client('sts')
Creds = stsClient.assume_role(
    RoleArn=RoleArn,
    RoleSessionName='CW-TO-ES-INGEST',
    DurationSeconds=900
)
AccessKey = ""
AccessKey = Creds['Credentials']['AccessKeyId']
SecretAccessKey = ""
SecretAccessKey = Creds['Credentials']['SecretAccessKey']
SessionToken = ""
SessionToken = Creds['Credentials']['SessionToken']

awsauth = AWS4Auth(AccessKey,SecretAccessKey,'us-east-1', 'es',session_token=SessionToken)

es = Elasticsearch(
    hosts=[{'host': host, 'port': 443}],
    http_auth=awsauth,
    use_ssl=True,
    verify_certs=True,
    connection_class=RequestsHttpConnection,
    timeout=40
)
print(es.info())

###################################################################################
### START FOREACH ACCOUNT / REGION LOOP HERE
###################################################################################

##  Loop through each account entry of the Accounts file we read in and converted from JSON earlier
for item in accounts:

    ##  Extract the AWS Account Number and Name from the item passed in from Accounts list (In the format of "account_number:account_name")
    Account = ""
    AccountName = ""
    item = item.split(":")
    Account = item[0]
    #print(Account)
    AccountName = item[1]
    AccountName = AccountName.lower()
    Division = ""
    if("<SOME_TERM>" in AccountName):
        Division = "<DIVISION_NAME>"
    elif("<SOME_OTHER_TERM>" in AccountName):
        Division = "<ANOTHER_DIVISION_NAME>"
    ## Continue this for each division type
    else:
        Division = "UNKNOWN"


    ##  Try to assume the <ROLE_NAME_TO_ASSUME> for the given account we are looping through using the AWS STS boto3 client
    ##  These temp STS credentials will then be used for our CloudWatch client to pull metrics for the given account we are looping through
    try:
        RoleArn = 'arn:aws:iam::' + Account + ':role/<ROLE_NAME_TO_ASSUME>'
        if Account == thisaccount:
            print("%s ACCOUNT - NOT USING STS" % thisaccount)
        else:
            Creds = stsClient.assume_role(
                RoleArn=RoleArn,
                RoleSessionName='Gather-Metrics'
            )
            AccessKey = ""
            AccessKey = Creds['Credentials']['AccessKeyId']
            SecretAccessKey = ""
            SecretAccessKey = Creds['Credentials']['SecretAccessKey']
            SessionToken = ""
            SessionToken = Creds['Credentials']['SessionToken']
    except:
        print("Error Getting Temp Creds From Account %s" % AccountName)


    ##  Loops through each region in the RegionsArray list specified at the top of the script
    for Region in RegionsArray:
        try:
            ##  If the account being compared is the thisaccount account DONT use STS as we already have credentials for this account,
            ##  otherwise create a CloudWatch AWS boto3 client for the given account and region
            if Account == thisaccount:
                cloudwatch_client = boto3.client('cloudwatch', region_name='us-east-1')
            else:
                cloudwatch_client = boto3.client('cloudwatch', aws_access_key_id=AccessKey, aws_secret_access_key=SecretAccessKey, aws_session_token=SessionToken, region_name=Region)
        except:
            print("ERROR CREATING CLOUDWATCH BOTO3 CLIENT IN SOME REGION")


        ##  Calls the CloudWatch Get Metric Data boto3 API call for the given account and region in the loop
        response = cloudwatch_client.get_metric_data(
            MetricDataQueries=[
                ## ALL Metrics To Retrieve (Each Line is a new Metric) 

                ##FOREACH METRIC:  {'Id':'metriclabel','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'metricname','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'metriclabel','ReturnData':True},
                {'Id':'oldaccesskeysdisable','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'Old-Access-Keys-Disable','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'oldaccesskeysdisable','ReturnData':True},
                {'Id':'appelbupdateoldssl','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'App-ELB-Update-Old-SSL','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'appelbupdateoldssl','ReturnData':True},
                {'Id':'unencrypteddynamodbtabledeleted','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'Unencrypted-DynamoDB-Table-Deleted','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'unencrypteddynamodbtabledeleted','ReturnData':True},
                {'Id':'unencryptedec2terminatedbypolicy','MetricStat':{'Metric':{'Namespace':'<YOUR_CW_METRIC_NAMESPACE>','MetricName':'Unencrypted-EC2-Terminated-By-Policy','Dimensions':[]},'Period':1800,'Stat':'Sum','Unit':'Count'},'Label':'unencryptedec2terminatedbypolicy','ReturnData':True}
            ],
            StartTime=fourtyFiveMinutesAgo,
            EndTime=fifteenMinutesAgo,
            MaxDatapoints=100800
        )

        ## For each metric in the returned CloudWatch get_metric_data command, loop through and extract the Label and Values and reset the counters
        for metric in response['MetricDataResults']:
            Label = ""
            Label = metric['Label']
            Values =""
            Values = metric['Values']
            Timestamp = datetime.now()
            Timestamp = metric['Timestamps']
            if not Timestamp:
                Timestamp = datetime.now()
   
            TotaloldaccesskeysdisableValuesByAccount = 0
            TotalappelbupdateoldsslValuesByAccount = 0
            TotalunencrypteddynamodbtabledeletedValuesByAccount = 0
            Totalunencryptedec2terminatedbypolicyValuesByAccount = 0
             
            ##print(Label)
            dt = datetime.now()
            microseconds = dt.microsecond
            epoch_time = int(time.time())
            randominteger = random.randint(1000, 9999)
            id=("%s%s%s" % (epoch_time,microseconds,randominteger))
            ## Sleep for a fraction of a second so that our microseconds timestamp is unique appending random int to ensure uniqueness
            time.sleep(1/100000.0)
            
            ## Based on the Label of the Metric being returned, sum it, print it and add it as a row in the csv file data variable
            
            #FOREACH IF..ELIF Label=MetricLabel
            if Label == 'oldaccesskeysdisable':
                for Value in Values:
                    dt = datetime.now()
                    microseconds = dt.microsecond
                    epoch_time = int(time.time())
                    randominteger = random.randint(1000, 9999)
                    id=("%s%s%s" % (epoch_time,microseconds,randominteger))
                    try:
                        TotaloldaccesskeysdisableValues = ( TotaloldaccesskeysdisableValues + Value )
                        TotaloldaccesskeysdisableValuesByAccount = ( TotaloldaccesskeysdisableValuesByAccount + Value )
                    except:
                        print("Error getting or adding Total oldaccesskeysdisable Values")
                if len(Values) > 0:
                    print("Account:  %s  Region:  %s  Label:  %s  Total Invocations:  %s" % (Account,Region,Label,int(TotaloldaccesskeysdisableValuesByAccount)))
                    oldaccesskeysdisableList = []
                    oldaccesskeysdisableList = [Account,AccountName,Region,Label,int(TotaloldaccesskeysdisableValuesByAccount)]    
                    doc = {
                            'MetricName': 'Old-Access-Keys-Disable',
                            'id': id,
                            'Policy': '',
                            'Resource': '',
                            'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
                            'Sum': TotaloldaccesskeysdisableValuesByAccount,
                            'Account': Account,
                            'AccountName': AccountName,
                            'Division': Division,
                            'Region': Region,
                            'timestamp': Timestamp,
                        }
                else:
                    doc = {
                            'MetricName': 'Old-Access-Keys-Disable',
                            'id': id,
                            'Policy': '',
                            'Resource': '',
                            'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
                            'Sum': 0,
                            'Account': Account,
                            'AccountName': AccountName,
                            'Division': Division,
                            'Region': Region,
                            'timestamp': Timestamp,
                        }
                docs.append(doc)
            elif Label == 'appelbupdateoldssl':
                for Value in Values:
                    dt = datetime.now()
                    microseconds = dt.microsecond
                    epoch_time = int(time.time())
                    randominteger = random.randint(1000, 9999)
                    id=("%s%s%s" % (epoch_time,microseconds,randominteger))
                    try:
                        TotalappelbupdateoldsslValues = ( TotalappelbupdateoldsslValues + Value )
                        TotalappelbupdateoldsslValuesByAccount = ( TotalappelbupdateoldsslValuesByAccount + Value )
                    except:
                        print("Error getting or adding Total appelbupdateoldssl Values")
                if len(Values) > 0:
                    print("Account:  %s  Region:  %s  Label:  %s  Total Invocations:  %s" % (Account,Region,Label,int(TotalappelbupdateoldsslValuesByAccount)))
                    appelbupdateoldsslList = []
                    appelbupdateoldsslList = [Account,AccountName,Region,Label,int(TotalappelbupdateoldsslValuesByAccount)]    
                    doc = {
                            'MetricName': 'App-ELB-Update-Old-SSL',
                            'id': id,
                            'Policy': '',
                            'Resource': '',
                            'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
                            'Sum': TotalappelbupdateoldsslValuesByAccount,
                            'Account': Account,
                            'AccountName': AccountName,
                            'Division': Division,
                            'Region': Region,
                            'timestamp': Timestamp,
                        }
                else:
                    doc = {
                            'MetricName': 'App-ELB-Update-Old-SSL',
                            'id': id,
                            'Policy': '',
                            'Resource': '',
                            'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
                            'Sum': 0,
                            'Account': Account,
                            'AccountName': AccountName,
                            'Division': Division,
                            'Region': Region,
                            'timestamp': Timestamp,
                        }
                docs.append(doc)
            elif Label == 'unencrypteddynamodbtabledeleted':
                for Value in Values:
                    dt = datetime.now()
                    microseconds = dt.microsecond
                    epoch_time = int(time.time())
                    randominteger = random.randint(1000, 9999)
                    id=("%s%s%s" % (epoch_time,microseconds,randominteger))
                    try:
                        TotalunencrypteddynamodbtabledeletedValues = ( TotalunencrypteddynamodbtabledeletedValues + Value )
                        TotalunencrypteddynamodbtabledeletedValuesByAccount = ( TotalunencrypteddynamodbtabledeletedValuesByAccount + Value )
                    except:
                        print("Error getting or adding Total unencrypteddynamodbtabledeleted Values")
                if len(Values) > 0:
                    print("Account:  %s  Region:  %s  Label:  %s  Total Invocations:  %s" % (Account,Region,Label,int(TotalunencrypteddynamodbtabledeletedValuesByAccount)))
                    unencrypteddynamodbtabledeletedList = []
                    unencrypteddynamodbtabledeletedList = [Account,AccountName,Region,Label,int(TotalunencrypteddynamodbtabledeletedValuesByAccount)]    
                    doc = {
                            'MetricName': 'Unencrypted-DynamoDB-Table-Deleted',
                            'id': id,
                            'Policy': '',
                            'Resource': '',
                            'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
                            'Sum': TotalunencrypteddynamodbtabledeletedValuesByAccount,
                            'Account': Account,
                            'AccountName': AccountName,
                            'Division': Division,
                            'Region': Region,
                            'timestamp': Timestamp,
                        }
                else:
                    doc = {
                            'MetricName': 'Unencrypted-DynamoDB-Table-Deleted',
                            'id': id,
                            'Policy': '',
                            'Resource': '',
                            'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
                            'Sum': 0,
                            'Account': Account,
                            'AccountName': AccountName,
                            'Division': Division,
                            'Region': Region,
                            'timestamp': Timestamp,
                        }
                docs.append(doc)
            elif Label == 'unencryptedec2terminatedbypolicy':
                for Value in Values:
                    dt = datetime.now()
                    microseconds = dt.microsecond
                    epoch_time = int(time.time())
                    randominteger = random.randint(1000, 9999)
                    id=("%s%s%s" % (epoch_time,microseconds,randominteger))
                    try:
                        Totalunencryptedec2terminatedbypolicyValues = ( Totalunencryptedec2terminatedbypolicyValues + Value )
                        Totalunencryptedec2terminatedbypolicyValuesByAccount = ( Totalunencryptedec2terminatedbypolicyValuesByAccount + Value )
                    except:
                        print("Error getting or adding Total unencryptedec2terminatedbypolicy Values")
                if len(Values) > 0:
                    print("Account:  %s  Region:  %s  Label:  %s  Total Invocations:  %s" % (Account,Region,Label,int(Totalunencryptedec2terminatedbypolicyValuesByAccount)))
                    unencryptedec2terminatedbypolicyList = []
                    unencryptedec2terminatedbypolicyList = [Account,AccountName,Region,Label,int(Totalunencryptedec2terminatedbypolicyValuesByAccount)]    
                    doc = {
                            'MetricName': 'Unencrypted-EC2-Terminated-By-Policy',
                            'id': id,
                            'Policy': '',
                            'Resource': '',
                            'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
                            'Sum': Totalunencryptedec2terminatedbypolicyValuesByAccount,
                            'Account': Account,
                            'AccountName': AccountName,
                            'Division': Division,
                            'Region': Region,
                            'timestamp': Timestamp,
                        }
                else:
                    doc = {
                            'MetricName': 'Unencrypted-EC2-Terminated-By-Policy',
                            'id': id,
                            'Policy': '',
                            'Resource': '',
                            'NameSpace': '<YOUR_CW_METRIC_NAMESPACE>',
                            'Sum': 0,
                            'Account': Account,
                            'AccountName': AccountName,
                            'Division': Division,
                            'Region': Region,
                            'timestamp': Timestamp,
                        }
                docs.append(doc)
            

            

## Print Totals across all accounts and specified regions for each policy metric
print(" ")
print(" ")
print(" ")

##FOREACH METRIC DO: print(" Total metriclabel Policy Invocations:  %s" % TotalmetriclabelValues)
print(" Total oldaccesskeysdisable Policy Invocations:  %s" % TotaloldaccesskeysdisableValues)
print(" Total appelbupdateoldssl Policy Invocations:  %s" % TotalappelbupdateoldsslValues)
print(" Total unencrypteddynamodbtabledeleted Policy Invocations:  %s" % TotalunencrypteddynamodbtabledeletedValues)
print(" Total unencryptedec2terminatedbypolicy Policy Invocations:  %s" % Totalunencryptedec2terminatedbypolicyValues)



bulk_action = []


print("THERE ARE %s  DOCS IN THE DOCS ARRAY" % (len(docs)))

for doc in docs:
    #print(doc)
    bulk = {
        "_index"  : '<YOUR_ELASTICSEARCH_INDEX_NAME>',
        "_type"   : 'metrics',
        "_id"     : doc['id'],
        "_source" : doc,
        }
    bulk_action.append(bulk)
try:
    helpers.bulk(es, bulk_action)
    print('Imported metrics data successfully!!!')
except Exception as ex:
    print('Error:', ex)

## Calculates Total Script Run Time            
TotalTime = (datetime.now() - ScriptStartTime)

## Print Script runtime
print(" ")
print(" ")
print("Script Finished.  Elaspsed Time: %s" % TotalTime)