Works slower than expected due to the unexpected uselessness of S3FS.

gistfile1.txt

#!usr/bin/python

#run: python S3UploadGit.py <full_repo> <s3_full_name>

from CREDENTIALS import AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from fabric.context_managers import shell_env
from fabric.api import local,settings,lcd,prefix
import os

def UploadToS3FromGit(repo,bucket):
    """ Upload a repo straight to S3, first creates and then un-creates an S3 linking on the machine in question in mount.
    """
    ##First we must check if the bucket exists.
    

    conn = S3Connection(AWS_ACCESS_KEY,AWS_SECRET_ACCESS_KEY)
    
    try:
        upload_bucket = conn.create_bucket(bucket)
    except Exception, e:
        print "Exception '%s' on bucket create" % e
    
    installed = True
    local("apt-get update")
    local("apt-get install build-essential libfuse-dev fuse-utils libcurl4-openssl-dev libxml2-dev mime-support -y")
    
    with settings(warn_only=True):
        
        local("mkdir /mnt/%s/" % (bucket))
        local("umount /mnt/%s/" % (bucket))
        if local("s3fs --version").return_code == 127:
            installed = False
            print "S3FS not built yet."
            print "Building S3FS, might take awhile the first time."
    if installed == False:
        local("rm -rf /tmp/rational/")
        local("wget http://s3fs.googlecode.com/files/s3fs-1.69.tar.gz -P /tmp/rational/")
        with lcd("/tmp/rational/"):
            local("tar -zxvf s3fs-1.69.tar.gz")
        with lcd("/tmp/rational/s3fs-1.69/"):
            from time import sleep
            local("./configure --prefix=/usr")
            local("make")
            local("make install")
    print "Creating Bucket Connection to S3"
    local("chmod 600 %s/keys/*" % os.getcwd())
    local("s3fs %s /mnt/%s/ -o passwd_file=%s/keys/s3_key" % (bucket, bucket,os.getcwd()))
    print "S3 Ready."
    #Git checkout.
    #Checkout a GIT Address straight to the folder.
    #Uses the deploy key supplied.
    print "Cloning"
    with settings(warn_only=True):
        local("rm -rf /tmp/%s " % bucket)
    with settings(warn_only=True):
        if local("ssh-agent bash -c 'ssh-add %s/keys/id_rsa;git clone %s /mnt/%s/'" % (os.getcwd(),repo,bucket)).return_code == 128:
            with lcd("/mnt/%s/" % bucket):
                if local("ssh-agent bash -c 'ssh-add %s/keys/id_rsa;git fetch'" % (os.getcwd())).return_code == 128:
                    #Delete the folder and re-clone
                    local("rm -rf *")
                    local("rm -rf .*")
                    local("ssh-agent bash -c 'ssh-add %s/keys/id_rsa;git clone %s /mnt/%s/'" % (os.getcwd(),repo,bucket))
    local("umount /mnt/%s/" % (bucket))
    
    
    with settings(warn_only=True):
        #CLEAN UP
        local("rm -rf /tmp/%s " % bucket)
        local("rm -rf /tmp/rational/")
    print "Success, well hopefully."
    return 1
            
if __name__ == "__main__":
    from sys import argv
    UploadToS3FromGit(argv[1],argv[2])