strombergdev

class S3:
    def __init__(self):
        self.s3 = boto3.resource('s3')
        self.config = TransferConfig(multipart_threshold=1024 * 25,
                                     max_concurrency=10,
                                     multipart_chunksize=1024 * 25,
                                     use_threads=True)

    def file_exists(self, path, bucket):
        file_size = path.stat().st_size

strombergdev

class Crypto:
    def __init__(self):
        self.buffersize = 64 * 1024

    @staticmethod
    def generate_key():
        return b64encode(os.urandom(32)).decode('utf-8')

    def encrypt_file(self, path, dest, key):
        with open(path, "rb") as fIn:

strombergdev

misc_files_archive = None

for item in path.iterdir():
    if item.is_dir():
        shutil.make_archive(
            base_name=temp_zips_folder.joinpath(item.name),
            format='zip',
            root_dir=str(item.parent),
            base_dir=str(item.name))

strombergdev

objects = s3.Bucket(bucket_name).objects.all()
    for object in objects:

strombergdev

def generate_s3_url(bucket, file_path):
    return s3.meta.client.generate_presigned_url('get_object',
                                                 Params={'Bucket': bucket,
                                                         'Key': file_path})


def create_folder(name, parent_asset_id):
    return frameio_client.create_asset(
        parent_asset_id=parent_asset_id,
        name=name,

strombergdev

def copy(event, context):
    """Copy files and folders from S3 to Frame.io. Restart if Lambda timeouts"""
    bucket_name = event['bucket_name']
    previous_asset = event['previous_asset']
    continued_run = event['continued_run']
    parent_ids = event['parent_ids']

    start_time = time()
    previous_folder = ''

strombergdev

lambda_client.invoke(
    FunctionName='s3-to-frameio-lambda-copy-dev-copy',
    InvocationType='Event',
    Payload=json.dumps({
        'bucket_name': bucket,
        'previous_asset': '',
        'continued_run': 'false',
        'parent_ids': {'': root_asset_id}})     # Dict with folder:asset_id pairs. '' is root.
)

strombergdev

service: s3-to-frameio-lambda-copy
frameworkVersion: '2'

provider:
  name: aws
  runtime: python3.8
  memorySize: 1024
  timeout: 900
  region: us-east-1
  iamRoleStatements:

strombergdev

Policy
Uploads

- Local files are considered ready for upload if their size hasn't changed in the last 60 secs.
- If the local xxhash does not match Frame.io's hash, the asset is deleted on Frame.io and re-uploaded.
- Assets are deleted and re-uploaded 3 times max.

Downloads

- Frame.io assets are considered ready for download when upload_completed_at is not None.

strombergdev

import frameioclient
from frameioclient.next_uploader import \
    NextFrameioUploader  # Rename the uploader class in next file...
from frameioclient.py3_uploader import FrameioUploader
import mimetypes
import os
import xxhash
from time import sleep

KB = 1024