Lambda to save Github webhook payload to s3

PY-GitHubWebHookToS3.py

# Handle payload from GitHub and write to S3 (for future consumption by Splunk)
#   - validates payload using key supplied at webhook creation
#   - only supports 'branch and tag' webhook from GitHub 
#   - requires api gateway
#   - uses Parameter store to retrieve environment specific details
#   - adds details to the payload (ishotfix)

# Could use a re-write especially if being used as an example because the code is fairly shoddy!

import os
import sys
import json
import hashlib
import hmac
import boto3
from botocore.exceptions import ClientError
import logging
import requests # Requires Lambda layer
from requests import Session, HTTPError

logger = logging.getLogger()
logger.setLevel(logging.INFO)

def getMetadata(event, body):
    # Inspect payload & ensure it contains supported keys or values
    if 'ref_type' not in body or (body['ref_type'] != 'tag' and body['ref_type'] != 'branch'):
        return { 'statusCode': 412, 'body': "Not implemented"}

    payloadMeta = {}

    try:
        payloadMeta['env'] = event['requestContext']['stage']
        payloadMeta['org'] = body['repository']['owner']['login']
        payloadMeta['repo'] = body['repository']['name']
    except KeyError as ex:
        return { 'statusCode': 400, 'body': 'Unable to find key: ' + ex }

    return payloadMeta

def getAPIKey():
    ssm = boto3.client('ssm')

    ssmPath = '{}GitHub/API'.format(os.environ['APP_CONFIG_PATH'])
    logger.info(ssmPath)
    parameter = ssm.get_parameter(Name=ssmPath, WithDecryption=True)
    return parameter['Parameter']['Value']

def getGHKey(meta):
    ssm = boto3.client('ssm')

    ssmPath = '{}GitHub/{}/{}/WebHookSecret'.format(os.environ['APP_CONFIG_PATH'], meta['org'], meta['repo'])
    logger.info(ssmPath)
    parameter = ssm.get_parameter(Name=ssmPath, WithDecryption=True)
    return parameter['Parameter']['Value']

# Veriffy the signature hash from the incomming headers and payload vs our config
def verify_signature(secret, signature, payload):
    computed_hash = hmac.new(secret.encode('ascii'), payload.encode('utf-8'), hashlib.sha1) # added .encode('utf-8') for lambda proxy
    computed_signature = '='.join(['sha1', computed_hash.hexdigest()])
    return hmac.compare_digest(computed_signature.encode('ascii'), signature.encode('ascii'))

def uploadJsonToS3(file_data, bucket, key):
    # Upload the file
    s3_client = boto3.client('s3')
    try:
        response = s3_client.put_object(
            Body=str(json.dumps(file_data)),
            Bucket=bucket,
            Key=key)
    except ClientError as ea:
        logging.error(ea)
        return False
    return True

def addMetaToPayload(payload):

    splunkConf = {}
    splunkConf['index'] = "devops"
    splunkConf['sourcetype'] = "github:git:" + payload['ref_type']
    splunkConf['source'] = "mygithubgobbler"

    updatedPL = {}
    updatedPL["metadata"] = splunkConf
    updatedPL["eventTimePath"] = ["commit","author","date"]
    updatedPL["events"] = [ payload ]

    return updatedPL

def setAdditionalTagFields(inputPayload):
    
    # Get version ID from payload, then create extra dict params
    isHotfix = False
    try:
        ver = inputPayload["ref"].split('-')[1]
    except:
        ver = inputPayload["ref"]

    if (ver.split('.')[-1] != "0"): # Any version not ending in 0 is a hotfix
        isHotfix = True
        print('its a hotfix')

    # Populate payload with commit details
    inputPayload["commit"]   = getCommitFromTag(inputPayload["ref"], inputPayload['repository']['url'])
    inputPayload["version"]  = ver
    inputPayload["name"]     = inputPayload["ref"]
    inputPayload["ishotfix"] = isHotfix

    return inputPayload

def getCommitFromTag(tagName, url):

    tagRefUrl = url + "/git/refs/tags/" + tagName
    logger.info("Looking up tag " + tagRefUrl)
    token =  getAPIKey()
    headers = { 'Authorization' : 'token  ' + token }

    try:
        tagLookup = requests.get(tagRefUrl, headers = headers)
        logger.info(tagLookup.json())
        tagLookup.raise_for_status()
        commit = requests.get(tagLookup.json()['object']['url'], headers = headers)
        commit.raise_for_status()
    except requests.exceptions.RequestException as e: 
        logger.error("error: {}".format(e))
        return { 'statusCode': 400, 'body': 'Unable to lookup tag commit' }
    return commit.json()

def lambda_handler(event, context):
    logger.info('event: ' + json.dumps(event))
    payloadBody = json.loads(event['body'])
    eventMeta = getMetadata(event, payloadBody)
    logger.info(eventMeta)
    if 'statusCode' in eventMeta:
        # StatusCode within eventMeta suggests we need to return
        return eventMeta
    apikey = getGHKey(eventMeta)

    verified = verify_signature(apikey, event['headers']['X-Hub-Signature'], event['body'])

    if verified:
        logger.info("Payload verified")
        updatedPayload = addMetaToPayload(payloadBody)
        if updatedPayload['events'][0]['ref_type'] == 'tag':
            updatedPayload['events'][0] = setAdditionalTagFields(updatedPayload['events'][0])

        logger.info("payload: ")
        logger.info(updatedPayload)
        
        s3key = "github/{}/{}/{}/{}.json".format(eventMeta['env'], eventMeta['org'], eventMeta['repo'], event['headers']['X-GitHub-Delivery'].lower())
        uploadReturn = uploadJsonToS3(updatedPayload, os.environ['spl_key'], s3key) 
    else:
        return { 'statusCode': 401, 'body': 'Not Authorized'}        

    return { 'statusCode' : 200, 'body' : uploadReturn }