A script to download and concatenate AWS ELB Access logs

s3downloader.py

import os
import pytz
import sys
import datetime
import dateutil.parser
import boto

downloadFolder = "downloads"

if len(sys.argv) > 1:
    downloadFolder = sys.argv[1]

if(not downloadFolder.endswith("/")):
    downloadFolder = downloadFolder + "/"

def getS3Files(bucketName):
    lastRunTime = getLastRunTime(bucketName)
    print(lastRunTime)

    c = boto.connect_s3(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
    bucket = c.get_bucket(bucketName)
    for key in bucket:
        keyDate = dateutil.parser.parse(key.last_modified)
        if ".log" in key.name and keyDate > lastRunTime:
            logFileName = str(keyDate.date()) + ".log"

            if not os.path.exists(downloadFolder):
                os.makedirs(downloadFolder)

            print("Appending " + key.name)
            with open(downloadFolder + logFileName, "a") as logFile:
                logFile.write(key.get_contents_as_string())
            setLastRunTime(bucketName, keyDate)

def getLastRunTime(bucketName):
    lastRunTime = datetime.datetime(2011,1,1,0,0,0,0,tzinfo=pytz.UTC)

    if os.path.exists("lastruntime." + bucketName + ".txt"):
        with open("lastruntime." + bucketName + ".txt", "r") as lastRunFile:
            lastRunTime = dateutil.parser.parse(lastRunFile.read())

    return lastRunTime

def setLastRunTime(bucketName, lastRunDate):
    with open("lastruntime." + bucketName + ".txt", "w") as lastRunFile:
        lastRunFile.write(lastRunDate.isoformat())
        
        
AWS_ACCESS_KEY = "ABCDEFG"
AWS_SECRET_ACCESS_KEY = "SUPER_SECRET_KEY"

getS3Files('bucket1')
getS3Files('bucket2')