lb_accesslog_partitioner.py

import boto3
import json
from datetime import datetime
from datetime import timedelta

# default_path : AWSLogs/111111111111/elasticloadbalancing/ap-northeast-2/2023/01/01/객체명
# ALB key : 111111111111_elasticloadbalancing_ap-northeast-2_app.pub-alb.82eb6fb7f7a90413_20210609T0000Z_10.10.17.197_ok8s9v7n.log.gz
# CLB key : 111111111111_elasticloadbalancing_ap-northeast-2_pub-clb_20230102T0300Z_10.10.2.125_15h4vyln.log

def lambda_handler(event, context):
    client = boto3.client('s3')

    for record in event['Records']:
        bucket_name = record['s3']['bucket']['name']
        default_path = record['s3']['object']['key']
        object_name = default_path.rsplit('/', 1)[-1]
        
        # get items with localizing timezone
        splitted_list = object_name.split('_')
        account_id = splitted_list[0]
        service = splitted_list[1]
        region = splitted_list[2]
        lb_name = splitted_list[3]
        kst_date = datetime.strptime(splitted_list[4], "%Y%m%dT%H%MZ") + timedelta(hours=9)
        
        # Check if minute is '00'
        if splitted_list[4][-3:-1] == '00': # 20230106T19 00 Z
            kst_date -= timedelta(hours=1)
            
        year = kst_date.strftime("%Y") 
        month = kst_date.strftime("%m")
        day = kst_date.strftime("%d")
        hour = kst_date.strftime("%H")

        # localizing timezone in object name
        splitted_list[4] = kst_date.strftime("%Y%m%dT%H%M") + "+09:00"
        localized_object_name = '_'.join(map(str, splitted_list))
        partitioned_path = f"AWSLogs/{account_id}/{service}/{region}/{lb_name}/{year}/{month}/{day}/{hour}/{localized_object_name}"
        print(partitioned_path)

        response = client.copy_object(
            Bucket = bucket_name,
            Key = partitioned_path,
            CopySource = bucket_name + '/' + default_path
        )

        response = client.delete_object(
            Bucket = bucket_name,
            Key = default_path
        )