Skip to content

Instantly share code, notes, and snippets.

@mendhak
Created March 23, 2014 01:47
Show Gist options
  • Save mendhak/9717352 to your computer and use it in GitHub Desktop.
Save mendhak/9717352 to your computer and use it in GitHub Desktop.
A script to download and concatenate AWS ELB Access logs
import os
import pytz
import sys
import datetime
import dateutil.parser
import boto
downloadFolder = "downloads"
if len(sys.argv) > 1:
downloadFolder = sys.argv[1]
if(not downloadFolder.endswith("/")):
downloadFolder = downloadFolder + "/"
def getS3Files(bucketName):
lastRunTime = getLastRunTime(bucketName)
print(lastRunTime)
c = boto.connect_s3(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
bucket = c.get_bucket(bucketName)
for key in bucket:
keyDate = dateutil.parser.parse(key.last_modified)
if ".log" in key.name and keyDate > lastRunTime:
logFileName = str(keyDate.date()) + ".log"
if not os.path.exists(downloadFolder):
os.makedirs(downloadFolder)
print("Appending " + key.name)
with open(downloadFolder + logFileName, "a") as logFile:
logFile.write(key.get_contents_as_string())
setLastRunTime(bucketName, keyDate)
def getLastRunTime(bucketName):
lastRunTime = datetime.datetime(2011,1,1,0,0,0,0,tzinfo=pytz.UTC)
if os.path.exists("lastruntime." + bucketName + ".txt"):
with open("lastruntime." + bucketName + ".txt", "r") as lastRunFile:
lastRunTime = dateutil.parser.parse(lastRunFile.read())
return lastRunTime
def setLastRunTime(bucketName, lastRunDate):
with open("lastruntime." + bucketName + ".txt", "w") as lastRunFile:
lastRunFile.write(lastRunDate.isoformat())
AWS_ACCESS_KEY = "ABCDEFG"
AWS_SECRET_ACCESS_KEY = "SUPER_SECRET_KEY"
getS3Files('bucket1')
getS3Files('bucket2')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment