boto3, cloudwatch, events

get_log_events.py

""" Get the log events for a log group from CloudWatch """
import boto3
from datetime import datetime, timedelta
import pandas as pd

ACCOUNT = 'prod'
GROUP = '/aws/lambda/lambda-function-prod'
NUM_DAYS = 7
START_TIME = int((datetime.now() - timedelta(days=NUM_DAYS)).timestamp())

# Setup the logs client
session = boto3.session.Session(profile_name=ACCOUNT)
logs_client = session.client('logs')

def get_events():
    """ Get the log events """
    events = []
    kwargs = {}
    while True:
        response = logs_client.filter_log_events(
            logGroupName=group,
            startTime=START_TIME,
            **kwargs)

        events += response['events']
        if 'nextToken' not in response:
            break
        kwargs['nextToken'] = response['nextToken']
    return events

df = pd.DataFrame(get_events())
# Correct the data
df['ingestionTime'] = pd.to_datetime(df['ingestionTime'], unit='ms')
df['message'] = df['message'].apply(lambda x: x.strip())