Copy static files for all public guides from prod to dev.

copy_static_s3_dev.py

#!/usr/bin/env python
from django.core.management import setup_environ
import settings
import boto
setup_environ(settings)
from guidebook.models import *
from guidebook.storage.s3sync import uploadFolderToS3, downloadFolderFromS3
import argparse

def copy_s3_files(s3DownloadFolder,localTempFolder,s3UploadFolder,from_bucket='media.guidebook.com',to_bucket='media-dev.guidebook.com'):
    import boto
    import settings
    from guidebook.storage.s3sync import uploadFolderToS3, downloadFolderFromS3
    ignore_patterns = [r'^%s/hosted/' % s3DownloadFolder,r'\.syncTask$']
    s3_connection = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
    bucket1 = s3_connection.get_bucket(from_bucket)
    bucket2 = s3_connection.get_bucket(to_bucket)

    print 'download: %s/%s' % (from_bucket,s3DownloadFolder)
    print 'upload: %s/%s' % (to_bucket,s3UploadFolder)
    print 'temp: %s' % localTempFolder
    downloadFolderFromS3(s3DownloadFolder,localTempFolder, ignore_patterns=ignore_patterns,bucket=bucket1)
    uploadFolderToS3(localTempFolder, s3UploadFolder,bucket=bucket2)

parser = argparse.ArgumentParser(description='Copy files from one s3 bucket to another.')
parser.add_argument("s3Folder", help="The folder you want to copy from s3")
parser.add_argument("-source", help="Source S3 bucket, default is media.guidebook.com")
parser.add_argument("-to", help="To S3 bucket, default is media-dev.guidebook.com")
parser.add_argument("-temp", help="override the tmp folder location")
parser.add_argument("-uploadFolder", help="override the upload Folder. default is the same as the copied folder")
args = parser.parse_args()

s3DownloadFolder = args.s3Folder
localTempFolder = args.temp if args.temp else '%s%s' % (settings.TEMP_PATH, s3DownloadFolder)
s3UploadFolder = args.uploadFolder if args.uploadFolder else s3DownloadFolder
from_bucket = args.source if args.source else 'media.guidebook.com'
to_bucket = args.to if args.to else 'media-dev.guidebook.com'



guides = Guide.objects.filter(private=False, approved=True, status__in=['P','U']).order_by('id')


print guides.count()
for guide in guides:
    print guide.id
    s3DownloadFolder = "upload/%s/" % guide.id
    localTempFolder = '%s%s' % (settings.TEMP_PATH, s3DownloadFolder)
    s3UploadFolder = s3DownloadFolder
    from_bucket = 'media.guidebook.com'
    to_bucket = 'media-dev.guidebook.com'
    copy_s3_files(s3DownloadFolder,localTempFolder,s3UploadFolder,from_bucket,to_bucket)
    print '------'