s3_safe_sync

s3_safe_sync

AWS_ACCESS_KEY = 'XXXXXXXXXXX'
AWS_SECRET_KEY  = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXX'
 
 
import os
import random, string
import boto
from boto.s3.connection import S3Connection
import hashlib
from subprocess import call

name_bucket = 'pica8'
prefix = 'test1'
name_key_file = 'key.txt'
key_file_s3 = prefix + name_key_file
backup_folder = '/Backups/'
key_file = backup_folder + name_key_file
key_file_s3 = prefix+'/'+name_key_file


conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
bucket = conn.get_bucket(name_bucket)


#get size of folder or file on s3
#Example: print get_size(bucket, 'test_folder')
#Example: print get_size(bucket, 'test_folder/testfile.txt')
def get_size(bucket, prefix):
    size = 0
    for key in bucket.list(prefix):
        size += key.size
    return size


#get dictionary with md5sum of files on bucket on s3
#Example: get_md5_s3(bucket)[u'i-9a4b699c/duplicity-full.20140704T010001Z.vol1.difftar.gz']
def get_md5_s3(bucket, file_s3):
    bucket_dict = {}
    rs_keys = bucket.get_all_keys()
    for key_val in rs_keys:
        bucket_dict[key_val.name] = key_val.etag.rstrip('"').lstrip('"')
    try:
        print "Note: md5sum of key file on s3: "+bucket_dict[file_s3]
        return bucket_dict[file_s3]
    except:
        print "Note: Key file don't exist on s3"


#ge md5sum of local file
def get_md5(filePath):
    with open(filePath, 'rb') as fh:
        m = hashlib.md5()
        while True:
            data = fh.read(8192)
            if not data:
                break
            m.update(data)
        return m.hexdigest()

# Create key file
# Example: create_key_file('/Backups/key.txt')
def create_key_file(path):
    if not os.path.exists(path):
        f = open(path, 'w+')
        f.write('Don\'t remove and change the file  \n')
        f.write(''.join(random.sample(string.digits+string.letters, 48)) + '\n')
        f.close()
        print "Note: created file "+path


#run command for sync to s3
def sync_to_s3(backup_folder=backup_folder,name_bucket=name_bucket,prefix=prefix):
    call(["nice", "-n", "15", "/usr/bin/s3cmd", "sync", backup_folder, "s3://"+name_bucket+"/"+prefix+"/"])
    print "Note: Sync is done"


#try to get md5sum of file on s3 and if local file exist function compare md5sum both of files
def match_key_files(file_local, file_s3):
    try:
        md5_file_s3 = get_md5_s3(bucket, file_s3)
    except KeyError:
        return False
    if os.path.isfile(file_local):
        print "Note: md5sum of key file on local server: "+ get_md5(file_local)
    else:
        print "Note: key file on local server don't exist "
    if os.path.isfile(file_local) and get_md5(file_local) == md5_file_s3:
        print "Note: Key files match"
        return True
    else:
        return False


# main function
def main(bucket, prefix, key_file_s3, key_file):
    size_folder = get_size(bucket, prefix)
    print "size of folder on s3: ", size_folder
    if size_folder == 0:
        create_key_file(key_file)
        sync_to_s3()  # run rsync
        return 0
    else:
        if match_key_files(key_file, key_file_s3):
            print "Finished checking and running sync"
            sync_to_s3()  # run rsync
            return 0
        else:
            print "Error: key files don't match for local folder and for s3 folder or folder on s3 is not empty and don't have key file"
            pass  # sent alert



main(bucket, prefix, key_file_s3, key_file)