s3 sync

gistfile1.txt

import hashlib
import os
from datetime import datetime

import boto3

bucket = '...'
s3 = boto3.client('s3', aws_access_key_id='...', aws_secret_access_key='...')


class S3Sync:
    def __init__(self):
        self.remotes = None
        self.locals = []

    def sync(self):
        print('read remotes')
        self.remotes = s3.list_objects_v2(Bucket=bucket)['Contents']
        print('read locals')
        self.read_locals()
        max_local = max([os.path.getmtime(local) for local in self.locals])
        max_remote = max([datetime.timestamp(remote['LastModified']) for remote in self.remotes])
        if max_local > max_remote:
            print('up')
            self.up()
        else:
            print('down')
            self.down()

    def read_locals(self):
        for current, dirs, files in os.walk('.'):
            if '.\\.idea' in current:
                continue
            for fname in files:
                if current == '.':
                    self.locals.append(fname)
                else:
                    path = current[2:].replace('\\', '/')
                    self.locals.append('{}/{}'.format(path, fname))

    def down(self):
        for content in self.remotes:
            key = content['Key']
            if key.endswith('/'):
                continue
            parts = key.split('/')
            if len(parts) > 1:
                os.makedirs('/'.join(parts[:-1]), exist_ok=True)
            if os.path.isfile(key):
                if self.is_diff(key):
                    self.get(key, 'update local')
                else:
                    print('no local update for {}'.format(key))
            else:
                self.get(key, 'add to local')
                
        keys = [key['Key'] for key in self.remotes]
        remove = [path for path in self.locals if path not in keys]
        for path in remove:
            print('delete local {}'.format(path))
            os.remove(path)
        if remove:
            self.clean()

    @staticmethod
    def is_diff(key):
        with open(key, 'rb') as f:
            fb = f.read()
        local = hashlib.md5(fb).hexdigest()
        remote = s3.head_object(Bucket=bucket, Key=key)['ETag']
        return remote.replace('"', '') != local

    @staticmethod
    def get(key, msg):
        print('{} {}'.format(msg, key))
        obj = s3.get_object(Bucket=bucket, Key=key)
        with open(key, 'wb') as f:
            f.write(obj['Body'].read())

    @staticmethod
    def clean():
        for current, dirs, files in os.walk('.'):
            if '.\\.idea' in current:
                continue
            for dir_ in dirs:
                if not os.listdir(dir_):
                    print('delete local empty {}'.format(dir_))
                    os.rmdir(dir_)

    @staticmethod
    def put(path, msg):
        print('{} {}'.format(msg, path))
        with open(path) as f:
            content = f.read()
        s3.put_object(Bucket=bucket, Key=path, Body=content)

    def up(self):
        for path in self.locals:
            key = next((key for key in self.remotes if key['Key'] == path), None)
            if key:
                if self.is_diff(path):
                    self.put(path, 'update remote')
                else:
                    print('no remote update for {}'.format(path))
            else:
                self.put(path, 'add to remote')

        remove = [key['Key'] for key in self.remotes if key['Key'] not in self.locals]
        for key in remove:
            print('delete remote {}'.format(key))
            s3.delete_object(Bucket=bucket, Key=key)


if __name__ == '__main__':
    s3sync = S3Sync()
    s3sync.sync()