copy_to_s3

copy_to_s3.py

def copy_to_s3(
        bucket: Bucket,
        files_to_copy: List[str],
        row: NamedTuple,
        s3_artifact_path: str,
        process_func: Optional[Callable[[str, List, str, bool], None]]=None,
        reload: bool=False
):

    guid = row.file_location

    for s3_key in files_to_copy:
        file_name = s3_key.split('/')[-1]
        copy_source = {
            'Bucket': "<your-prod-s3-bucket>",
            'Key': s3_key
        }
        new_s3_key = f”{s3_artifact_path}{guid}/{file_name})”
        bucket.copy(copy_source, new_s3_key)
    if process_func:
        guid_prefix = f”{s3_artifact_path}{guid}”
        existing_files = list(bucket.objects.filter(Prefix=guid_prefix))
        process_func(row, existing_files, s3_artifact_path, reload)