Create a file that evenly assigns (by files) data from s3 across all tagged EC2 instances. Useful for a distributed database.

assigns3files.py

#!/usr/bin/env python

#boto libraries
import boto.ec2
import boto.utils
import boto.s3.connection

#python libraries
import json

AWS_ACCESS_KEY_ID = ''
AWS_ACCESS_KEY_SECRET = ''

S3_BUCKET_NAME = ''

with open('instance.json') as config_file:
    aws_auth = json.load(config_file)
    AWS_ACCESS_KEY_ID = aws_auth.get('access_key_id', '')
    AWS_ACCESS_KEY_SECRET = aws_auth.get('access_key_secret', '')
    S3_BUCKET_NAME = aws_auth.get('bucket', '')
    S3_DIRECTORY   = aws_auth.get('directory', '')

INSTANCE_ID = boto.utils.get_instance_metadata()['instance-id']
AWS_REGION = boto.utils.get_instance_identity()['document']['region']
print INSTANCE_ID > assignfiles.log
print AWS_REGION >> assignfiles.log

connection = boto.ec2.connect_to_region(AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_ACCESS_KEY_SECRET)
current_instance = connection.get_all_instances(instance_ids = [INSTANCE_ID])[0].instances[0]

name_tag = current_instance.tags['Name']
print name_tag >> assignfiles.log

group_instances = connection.get_all_instances(filters = {"tag:Name":name_tag})[0].instances
print group_instances >> assignfiles.log

connection = boto.s3.connection.S3Connection(AWS_ACCESS_KEY_ID, AWS_ACCESS_KEY_SECRET)
bucket = connection.get_bucket(S3_BUCKET_NAME)

keys = bucket.list(S3_DIRECTORY)

nodes = []
assignments = {}
for instance in sorted(group_instances):
    nodes.append(instance.id)
    assignments[instance.id] = []

print "creating assignations file" >> assignfiles.log
nodecount = len(nodes)
keycount = 0
for key in sorted(keys):
    if key.size > 0:
        index = keycount % nodecount
        instanceid = nodes[index]
        assignments[instanceid].append(key.name)
        keycount += 1

with open('ec2.assignations.json', 'w') as nodefile:
    json.dump( assignments, nodefile, sort_keys = True, indent = 4, separators =(',', ':') )