Skip to content

Instantly share code, notes, and snippets.

@hasantayyar
Created November 5, 2015 14:23
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save hasantayyar/a7d9da3840122c09feeb to your computer and use it in GitHub Desktop.
Save hasantayyar/a7d9da3840122c09feeb to your computer and use it in GitHub Desktop.
s3 to logentries aws lamda script
import json
import urllib
import boto3
import httplib
import re
import json
addr = '/%s/hosts/%s/%s?realtime=1' % ("accountkey","hostkey","logkey")
print('Loading function')
s3 = boto3.client('s3')
def lambda_handler(event, context):
print("Received event: " + json.dumps(event, indent=2))
# Get the object from the event and show its content type
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(event['Records'][0]['s3']['object']['key']).decode('utf8')
conn = httplib.HTTPConnection("api.logentries.com")
conn.request('PUT', addr )
conn.send('{ "level":"info","content": "Received Event"}\n')
conn.close()
try:
key = boto3.resource('s3').Object(bucket, key).get()
print("CONTENT TYPE: " + key['ContentType'])
chunk = key['Body'].read(1024*8)
while chunk:
logLines(chunk)
chunk = key['Body'].read(1024*8)
return key['ContentType']
except Exception as e:
print(e)
print('Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
raise e
def logLines(content):
conn = httplib.HTTPConnection("api.logentries.com")
conn.request('PUT', addr )
lines = content.splitlines()
for line in lines:
if(len(line) > 1):
pattern = '^([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*):([0-9]*) ([.0-9]*) ([.0-9]*) ([.0-9]*) (-|[0-9]*) (-|[0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) ([^ ]*) (- |[^ ]*)\" \"([^\"]*)\"'
d = re.split(pattern,line)
print(d)
if( d and len(d)>11 and (int(d[10])>390 or int(d[11])>390)):
message = { "elb_status_code": d[10], "backend_status_code": d[11], "method":d[14], "page":d[15], "user_agent": d[17],"client_ip":d[3],"server_ip":d[5], "request_processing_time":d[7],"backend_processing_time":d[8],"response_processing_time":d[9] }
conn.send(json.dumps(message)+"\n")
conn.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment