Scripts for packing and uploading AWS lambdas

indexer.py

import json
import tarfile
import urllib
from datetime import datetime

import boto3

s3_client = boto3.client('s3')
dynamo = boto3.resource('dynamodb')

def lambda_handler(event, _):
    """
    Handles event on a specified s3 bucket
    Args:
        event: (dict) event from the s3 service
        _: (dict) context of the event
    Returns:
        None
    """
    print("working")
    s3 = event['Records'][0]['s3']
    bucket = s3['bucket']['name']

    key = urllib.unquote(s3['object']['key'])
    print("Indexing " + bucket + "/" + key)
    if bucket == "bucket_name" and key.startswith("prefix_name"):
        db_item = create_db_item(bucket, key, event['Records'][0]['awsRegion'])
        print("Writing results to index")
        index_table = dynamo.Table(bucket + "-index")
    index_table.put_item(Item=db_item)


def create_db_item(bucket, key, event_region):
    """
    Creates item based on aws file 
    Args:
        bucket: (str) name of the bucket where file is located
        key: (str) name of the file
        event_region (str): region of the event

    Returns:
        db_item: (dict) object containing specified info
    """
    log_url = "https://" + bucket + ".s3-" + event_region \
              + ".amazonaws.com/" + key
    task, robot, ts = perception_regex.match(key.split('/')[-1]).groups()
    assert all(char.isdigit() or char == '.' for char in ts)
    assert task in {'barcode_search', 'estimate_putdown'}
    time = datetime.utcfromtimestamp(float(ts)).strftime('%Y-%m-%d %H:%M:%S')
    db_item = {"robot": robot,
               "timestamp": time,
               "task": task,
               "log_url": log_url}
    s3_client.download_file(bucket, key, '/tmp/file')
    with tarfile.open('/tmp/file', "r:gz") as tar:
        for member in tar.getmembers():
            file = tar.extractfile(member)
            file_name = member.name[:member.name.rfind('.')]
            if file_name == 'goal':
                db_item.update(parse_goal_log(json.loads(file.read())))
            if file_name == 'result':
                db_item.update(parse_result_log(json.loads(file.read())))
    return db_item

pack.sh

#!/usr/bin/env bash

rm ./function.zip
cd env/lib/python2.7/site-packages/
zip -r9 ${OLDPWD}/function.zip .
cd -
cd src
zip -g ${OLDPWD}/function.zip ./*
cd -

pack_and_deploy.sh

#!/usr/bin/env bash

./pack.sh
aws lambda update-function-code --zip-file fileb://function.zip --function-name S3-manipulation-log-index #  --dry-run

requirements.txt

boto3==1.9.153
botocore==1.12.153
docutils==0.14
futures==3.2.0
jmespath==0.9.4
pkg-resources==0.0.0
python-dateutil==2.8.0
s3transfer==0.2.0
six==1.12.0
urllib3==1.24.3
dill>=0.2.9
numpy>=1.16.3
pyyaml
rospkg

setup.sh

#!/usr/bin/env bash

virtualenv env
source env/bin/activate
pip install -r requirements.txt
deactivate