split_files.py

#!/usr/bin/env python

import os


def file_size_from_list(incoming_list):
    ## returns total size in bytes of all files in the incoming list

    total = 0

    for each_file in incoming_list:
        info = os.stat(each_file)
        each_file_size = info.st_size
        total += each_file_size

    return total


def split_files_on_size(incoming_files, size):
    ## 'incoming_files' is a list of strings of the paths to the files
    ## 'size' is a float/int of the number of megabytes to split at
    ## this will return a list of lists.

    size_in_bytes_cutoff = size * 1048576

    temp_total = 0

    output_parts = list()

    for each_file in incoming_files:
        info = os.stat(each_file)
        each_file_size = info.st_size
        if output_parts:
            size_of_last_set = file_size_from_list(output_parts[-1])
            if size_of_last_set + each_file_size <= size_in_bytes_cutoff:
                output_parts[-1].append(each_file)
            else:
                output_parts.append([each_file])
        else:
            output_parts.append([each_file])

    return output_parts


base_path = '/home/user/some_random_folder_with_lots_of_files/'
myfiles = os.listdir(base_path)

new_myfiles = list()

for x in myfiles:
    new_myfiles.append(base_path + x)

new_myfiles_split = split_files_on_size(new_myfiles, 10)

for x in new_myfiles_split:
    print file_size_from_list(x), x