Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
s3 to logentries aws lamda script
import json
import urllib
import boto3
import httplib
import re
import json
addr = '/%s/hosts/%s/%s?realtime=1' % ("accountkey","hostkey","logkey")
print('Loading function')
s3 = boto3.client('s3')
def lambda_handler(event, context):
print("Received event: " + json.dumps(event, indent=2))
# Get the object from the event and show its content type
bucket = event['Records'][0]['s3']['bucket']['name']
key = urllib.unquote_plus(event['Records'][0]['s3']['object']['key']).decode('utf8')
conn = httplib.HTTPConnection("api.logentries.com")
conn.request('PUT', addr )
conn.send('{ "level":"info","content": "Received Event"}\n')
conn.close()
try:
key = boto3.resource('s3').Object(bucket, key).get()
print("CONTENT TYPE: " + key['ContentType'])
chunk = key['Body'].read(1024*8)
while chunk:
logLines(chunk)
chunk = key['Body'].read(1024*8)
return key['ContentType']
except Exception as e:
print(e)
print('Error getting object {} from bucket {}. Make sure they exist and your bucket is in the same region as this function.'.format(key, bucket))
raise e
def logLines(content):
conn = httplib.HTTPConnection("api.logentries.com")
conn.request('PUT', addr )
lines = content.splitlines()
for line in lines:
if(len(line) > 1):
pattern = '^([^ ]*) ([^ ]*) ([^ ]*):([0-9]*) ([^ ]*):([0-9]*) ([.0-9]*) ([.0-9]*) ([.0-9]*) (-|[0-9]*) (-|[0-9]*) ([-0-9]*) ([-0-9]*) \"([^ ]*) ([^ ]*) (- |[^ ]*)\" \"([^\"]*)\"'
d = re.split(pattern,line)
print(d)
if( d and len(d)>11 and (int(d[10])>390 or int(d[11])>390)):
message = { "elb_status_code": d[10], "backend_status_code": d[11], "method":d[14], "page":d[15], "user_agent": d[17],"client_ip":d[3],"server_ip":d[5], "request_processing_time":d[7],"backend_processing_time":d[8],"response_processing_time":d[9] }
conn.send(json.dumps(message)+"\n")
conn.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.