Log files to Amazon S3

scriptchik.py

import os
import logging
from functools import partial
from itertools import chain, starmap

from boto3 import Session
from botocore.exceptions import ClientError

ACCESS_KEY = os.environ.get("ACCESS_KEY", "")
SECRET_KEY = os.environ.get("SECRET_KEY", "")
BUCKET_NAME = os.environ.get("BUCKET_NAME", "")
LOG_DIRS = (
    '/opt/data/web/',
    '/opt/data/db/'
)

logging.basicConfig(filename="error.log", format="%(levelname)s:%(asctime)s:%(message)s")
s3 = Session(ACCESS_KEY, SECRET_KEY).resource("s3")

try:
    s3.meta.client.head_bucket(Bucket=BUCKET_NAME)
except ClientError as ce:
    logging.warning(ce)
    bucket = s3.create_bucket(Bucket=BUCKET_NAME)
    logging.info("Bucket named {} is created".format(BUCKET_NAME))
else:
    bucket = s3.Bucket(BUCKET_NAME)


def joiner(files, dir):
    pj = partial(os.path.join, dir)
    return map(pj, files)

all(map(lambda f: bucket.put_object(Key=f, Body=f), chain(*starmap(joiner, zip([os.listdir(d) for d in LOG_DIRS], LOG_DIRS)))))