etag-save-to-s3.py

from datetime import datetime
import json
import boto3
import random

print('[INFO] Processing started')

random.seed(datetime.now())

dt = datetime.now()

BUCKET = 'ljmocic-test'
FOLDER_READ = 'etag_read/'
FOLDER_SAVE = 'etag_save/'
FILE_NAME = 'S3_cksum_' + str(dt.year) + '_' + str(dt.month) + '_' + str(dt.day) + '_' + str(random.randint(100000, 999999))
TEST_MODE = 0  # 1 - enabled, 0 - disabled


def lambda_handler(event, context):
    s3 = boto3.client('s3')

    save = []

    paginator = s3.get_paginator('list_objects')
    operation_parameters = {'Bucket': BUCKET,
                            'Prefix': FOLDER_READ}
    page_iterator = paginator.paginate(**operation_parameters)
    count = 0
    for page in page_iterator:
        #print(page['Contents'])
        for key in page['Contents']:
            if key['Key'].startswith(FOLDER_READ) == True and key['Key'].startswith(FOLDER_SAVE) == False and key['Key'].endswith('/') == False:
                #print(key['Key'])
                temp = {}
                temp[key['Key']] = key['ETag'][1:-1]
                save.append(temp)
        count += 1
        print('Current page: ' + str(count))
        print('Number of items: ' + str(len(save)))

    if TEST_MODE == 0:
        print('[INFO] Saving a file: ' + FOLDER_SAVE + FILE_NAME)
        #print('[INFO] File content: ' + str(save))
        s3.put_object(Key=FOLDER_SAVE + FILE_NAME, ContentType='application/json',
                      Body=json.dumps(save, indent=3), Bucket=BUCKET)
    elif TEST_MODE == 1:
        print('[TEST-MODE] Saving a file: ' + FOLDER_SAVE + FILE_NAME)
        #print('[TEST-MODE] File content: ' + str(save))

    print('[INFO] Processing finished')
    return {
        "statusCode": 200,
        "body": json.dumps('[INFO] Execution complete')
    }


if __name__ == "__main__":
    try:
        lambda_handler(None, None)
    except Exception as e:
        print(e)
        pass