unit_download_options.py

def get_unit_all_or_nothing(unit):
    """ 
    Tries to download all files associated with a unit. As soon as one fails,
    it aborts and abandons what has been downloaded so far.

    I believe this is the current aggreed-upon plan.
    """
    if unit.downloaded is True:
        # nothing to do then!
        return

    # This is the same in both cases.
    files = unit.somehow_determine_files()

    try:
        for f in files:
            download(f)
    except:
        # Anything that was downloaded gets abandoned, and later cleaned up.
        return

    # I assume this moves the files into their permanent location and sets the
    # "download" attribute on the unit to True
    unit.set_content()
    
    
def get_unit_partial_ok(unit):
    """ 
    Tries to download all files associated with a unit. Is tolerant of some
    files already existing.

    If multiple tasks end up trying to download the same unit at the same time,
    this approach reduces the chance of them doing double work, especially if
    the files are looped over in a non-deterministic order.

    A failure of one download does not prevent the other files from being put
    into their permanent location.
    """
    if unit.downloaded is True:
        # nothing to do then!
        return

    # This is the same in both cases.
    files = unit.somehow_determine_files()

    failure = False

    for f in files:
        try:
            # This could optionally verify the correct size as a cheap
            # integrity check.
            if not f.exists_on_disk:
                download(f)
                move_into_final_location(f)
        except:
            # remember the failure, but keep going
            failure = True
            continue

    # If one or more downloads failed, do not mark the unit as downloaded.
    if failure is False:
        unit.downloaded = True
        unit.save()