gistfile1.txt

#!/usr/bin/env python

import glob
import struct

from boto.s3.connection import S3Connection
from multiprocessing.pool import ThreadPool

def files():
    return glob.glob("tmp/inventory/*.gz")

def is_valid(file):
    with open(file, 'rb') as f:
        f.seek(-4, 2)
        return True if struct.unpack('I', f.read(4))[0] > 0 else False

def group_files(sequence, chunk_size):
    return list(zip(*[iter(sequence)] * chunk_size))

if __name__ == "__main__":
    for grouped_files in group_files(files(), 10):
        for file in grouped_files:
            if is_valid(file):
                print("Sending %s to s3" % (file))