poll_for_aws_upload.py

import os
import sys
import time

import requests

PREFIX = sys.argv[1]
AWS_ACCESS_KEY_ID = os.environ["AWS_ACCESS_KEY_ID"]
AWS_SECRET_ACCESS_KEY = os.environ["AWS_SECRET_ACCESS_KEY"]
TIMEZONE_KEY = os.environ["TIMEZONE_KEY"]
INPUT_DIR = "/mnt/in_memory_filesystem"
OUTPUT_BUCKET = os.environ["AWS_BUCKET"]
POLL_SECONDS = 20


def original_extension(fname):
    return fname.split(".")[-1]


def get_timestamp():
    ret = requests.get("http://api.timezonedb.com/v2/get-time-zone?key=%s&format=json&by=zone&zone=UTC" % TIMEZONE_KEY).json()["timestamp"]

    # ensure API restrictions aren't violated
    time.sleep(1)
    return ret


def get_files():
    return filter(
        lambda fname: fname.startswith("%s_chunk" % PREFIX),
        sorted(os.listdir(INPUT_DIR)),
    )[:-1]


while True:
    for fname in get_files():
        try:
            input_file = "%s/%s" % (INPUT_DIR, fname)
            os.system(
                "AWS_ACCESS_KEY_ID='%s' AWS_SECRET_ACCESS_KEY='%s' env/bin/aws s3 cp %s s3://%s/%s" % (
                    AWS_ACCESS_KEY_ID,
                    AWS_SECRET_ACCESS_KEY,
                    input_file,
                    OUTPUT_BUCKET,
                    "%s.%s" % (get_timestamp(), original_extension(input_file)),
                )
            )
            os.system("rm %s" % input_file)
            print "Uploaded %s" % input_file
        except Exception as e:
            print e
    time.sleep(POLL_SECONDS)