ELB LOG TO S3

README.md

ELB Classic 's log setting modify to enable.

boto3 sample code

docker-compose.yml

version: '3'
services: 
  app:
    build: ./app
    ports:
     - "8080:8080"
     - "8443:8443"
    volumes: 
      - ./app:/app
    # - /var/run/docker.sock:/var/run/docker.sock
    command: /works/code-server --auth none --host 0.0.0.0 --port 8443 /app 

Dockerfile

FROM python:3.8

RUN apt-get update
RUN apt-get install -y wget curl

WORKDIR /works
RUN wget https://github.com/cdr/code-server/releases/download/2.1688-vsc1.39.2/code-server2.1688-vsc1.39.2-linux-x86_64.tar.gz
RUN tar -xzf code-server2.1688-vsc1.39.2-linux-x86_64.tar.gz -C ./ --strip-components 1
RUN /works/code-server --install-extension ms-python.python

extract_ebs_log_line.py

import re
from typing import List

LOG_PARAM_NAME:List = [
    'ip','address','elb','client:port','backend:port',
    'request_processing_time','backend_processing_time',
    'response_processing_time','elb_status_code','backend_status_code',
    'received_bytes','sent_bytes','request','user_agent','ssl_cipher','ssl_protocol']

PARAM_TIMESTAMP = 0
PARAM_ELB = 1
PARAM_CLIENT_PORT = 2
PARAM_BACKEND_PORT = 3
PARAM_REQUEST_PROCESSING_TIME = 4
PARAM_BACKEND_PROCESSING_TIME = 5
PARAM_RESPONSE_PROCESSING_TIME = 6
PARAM_ELB_STATUS_CODE = 7
PARAM_BACKEND_STATUS_CODE = 8
PARAM_RECEIVED_BYTES = 9
PARAM_SENT_BYTES = 10
PARAM_REQUEST = 11
PARAM_USER_AGENT = 12
PARAM_SSK_CIPHER = 13
PARAM_SSL_PROTOCOL = 14

def extractLine(data:str):
    regex_src = r'^([^ ]+)'
    for _ in range(14):
        regex_src += r' ("[^"]+"|[^ \"]+)'
    regex=re.compile(regex_src+r'.*$') 
    res = regex.search(data)
    if res:
        return list(res.groups())
    else:
        return []

#
res = extractLine("<line data/>"))
res = dict(zip(LOG_PARAM_NAME,res))
print("<xx>{}</xx>".format(res))
#

main_setup_s3_and_elb.py

import botocore
import json

from boto3_type_annotations import s3
from boto3_type_annotations import elb

s3_client:s3.Client = boto3.client('s3')
elb_client:elb.Client = boto3.client('elb')
TARGET_BUCKET_NAME = "<your bucket>"
AWS_ACCOUNT_ID = "582318560864"#TOKYO
LOADBALANCER_NAME=""
REGION='ap-northeast-1'
# https://docs.aws.amazon.com/ja_jp/elasticloadbalancing/latest/classic/enable-access-logs.html#attach-bucket-policy

def create_s3_bucket():
    try:
        s3_client.create_bucket(
            Bucket=TARGET_BUCKET_NAME,
            CreateBucketConfiguration={
                'LocationConstraint': REGION
            })
    except botocore.exceptions.ClientError as e:
        print("{}".format(e.response))
        if e.response['Error']['Code'] == 'BucketAlreadyOwnedByYou':
            print("[W] Bucket is Already Exist!!")
        else:
            raise e

def get_policy():
    try:
        policy = s3_client.get_bucket_policy(Bucket=TARGET_BUCKET_NAME)
        print("{}".format(policy))
    except:
        print("[W] NOT FOUND POLICY")

def attach_policy():
    jsonDict = {
        "Version": "2012-10-17",
        "Statement": [
            {
            "Effect": "Allow",
            "Principal": {
                "AWS": "arn:aws:iam::{}:root".format(AWS_ACCOUNT_ID)
            },
            "Action": "s3:PutObject",
            "Resource": "arn:aws:s3:::{}/*/*".format(TARGET_BUCKET_NAME)
            }
        ]}
    jsonTxt = json.dumps(jsonDict)
    s3_client.put_bucket_policy(Bucket=TARGET_BUCKET_NAME,Policy=jsonTxt)

def set_log_enable():
    res = elb_client.describe_load_balancer_attributes(LoadBalancerName=LOADBALANCER_NAME)
    print(">A>{}".format(res))
    res['LoadBalancerAttributes']['AccessLog']['Enabled'] = True
    res['LoadBalancerAttributes']['AccessLog']['S3BucketName'] = TARGET_BUCKET_NAME
    res['LoadBalancerAttributes']['AccessLog']['S3BucketPrefix'] =  "elb"
    res['LoadBalancerAttributes']['AccessLog']['EmitInterval'] = 60 # 60min

    print(">B>{}".format(res))
    elb_client.modify_load_balancer_attributes(
        LoadBalancerName=LOADBALANCER_NAME,
        LoadBalancerAttributes=res['LoadBalancerAttributes'])



print("## CREATE")
create_s3_bucket()
get_policy()

print("## ATTACH")
attach_policy()
get_policy()


print("## LOG ENABLE")
set_log_enable()

requirements.txt

astroid==2.3.3
aws==0.2.5
awscli==1.16.278
bcrypt==3.1.7
boto==2.49.0
boto3==1.10.14
boto3-type-annotations==0.3.1
botocore==1.13.14
cffi==1.13.2
colorama==0.4.1
cryptography==2.8
docutils==0.15.2
fabric==2.5.0
freeze==1.0.10
invoke==1.3.0
isort==4.3.21
jmespath==0.9.4
lazy-object-proxy==1.4.3
mccabe==0.6.1
paramiko==2.6.0
prettytable==0.7.2
pyasn1==0.4.7
pycparser==2.19
pylint==2.4.3
PyNaCl==1.3.0
python-dateutil==2.8.0
PyYAML==5.1.2
rsa==3.4.2
s3transfer==0.2.1
six==1.13.0
urllib3==1.25.6
wrapt==1.11.2