clean_streams.py

#!/usr/bin/python3

import os
import glob

TARGET_SIZE = (1024**4) * 3
COLLECTION_CONFIGS = {
    'cam1': {
        'glob': '/media/camera/streams/cam1/cam1_*.mp4',
        'weight': 0.2,
    },
    'cam2': {
        'glob': '/media/camera/streams/cam2/cam2_*.mp4',
        'weight': 1.0,
    },
    'cam3': {
        'glob': '/media/camera/streams/cam3/cam3_*.mp4',
        'weight': 0.1,
    },
    'cam4': {
        'glob': '/media/camera/streams/cam4/cam4_*.mp4',
        'weight': 1.0,
    },
}

files = {}
file_total_sizes = {}
for collection in COLLECTION_CONFIGS:
    files[collection] = {}
    file_total_sizes[collection] = 0
    for fn in glob.glob(COLLECTION_CONFIGS[collection]['glob']):
        files[collection][fn] = os.stat(fn)
        file_total_sizes[collection] += files[collection][fn].st_size
    print('{}: {} files, {:0.02f} GiB, {:0.02f} weight'.format(
        collection,
        len(files[collection]),
        file_total_sizes[collection] / (1024**3),
        COLLECTION_CONFIGS[collection]['weight'],
    ))

grand_total_size = sum([file_total_sizes[collection] for collection in file_total_sizes])
print('Grand total: {:0.02f} GiB used, {:0.02f} GiB target, {:0.02f} GiB above target'.format(
    grand_total_size / (1024**3),
    TARGET_SIZE / (1024**3),
    (grand_total_size - TARGET_SIZE) / (1024**3),
))

round = 0
to_consider = list(COLLECTION_CONFIGS.keys())
consider_target_size = TARGET_SIZE
rescan = True
while rescan:
    rescan = False
    round += 1
    sum_weights = sum([COLLECTION_CONFIGS[collection]['weight'] for collection in to_consider])
    round_target_size = consider_target_size
    for collection in list(to_consider):
        collection_target_size = round_target_size * (COLLECTION_CONFIGS[collection]['weight'] / sum_weights)
        print('Round {}: {}: {:0.02f} GiB used, {:0.02f} GiB target, {:0.02%} round weight of {:0.02f} GiB, {:0.02f} GiB above target{}'.format(
            round,
            collection,
            file_total_sizes[collection] / (1024**3),
            collection_target_size / (1024**3),
            COLLECTION_CONFIGS[collection]['weight'] / sum_weights,
            round_target_size / (1024**3),
            (file_total_sizes[collection] - collection_target_size) / (1024**3),
            (' (disqualifying)' if file_total_sizes[collection] <= collection_target_size else ''),
        ))
        if file_total_sizes[collection] <= collection_target_size:
            consider_target_size -= file_total_sizes[collection]
            to_consider.remove(collection)
            rescan = True

sum_weights = sum([COLLECTION_CONFIGS[collection]['weight'] for collection in to_consider])
for collection in to_consider:
    collection_target_size = consider_target_size * (COLLECTION_CONFIGS[collection]['weight'] / sum_weights)
    to_free = file_total_sizes[collection] - collection_target_size
    print('{}: Freeing {:0.02f} GiB'.format(collection, to_free / (1024**3)))
    for fn in sorted(files[collection], key=lambda fn: files[collection][fn].st_mtime):
        to_free -= files[collection][fn].st_size
        print('Deleting {} ({:0.02f} MiB)'.format(
            fn,
            files[collection][fn].st_size / (1024**2),
        ))
        #os.remove(fn)
        if to_free <= 0:
            break