artifact.py

def _update_artifacts(location):
    artifact_config = '{0}/artifacts.json'.format(location)

    if os.path.exists(artifact_config):
        with open(artifact_config) as json_file:
            artifacts = json.load(json_file)

        for artifact in artifacts:
            if artifact['path'].startswith('/'):
                print('ERROR: Path {0} for artifact {1} must not be relative to this repository root.',
                    artifact['path'],
                    artifact['name']
                )
                return -1


            artifact_path = '{0}/{1}'.format(location, artifact['path'])
            artifact.setdefault('checksum', None)
            artifact.setdefault('checksum_algorithm', hashlib.sha256)

            # if the path exists, make sure the checksum matches
            if os.path.exists(artifact_path):
               pass 
            else:
                try:
                    download_file(
                        artifact['url'],
                        filename=artifact_path,
                        checksum=artifact['checksum'],
                        checksum_algorithm=artifact['checksum_algorithm']
                    )
                except Exception as e:
                    print('ERROR: Failed when downloading {0} to {1}. {2}'.format(
                        artifact['url'], artifact_path, e)
                    )


def checksum_file(file, checksum_algorithm=hashlib.sha256):
    if isinstance(file, str):
        file = open(file, 'r')

    if not callable(checksum_algorithm):
        checksum_algorithm = getattr(hashlib, checksum_algorithm)
    file_hash = checksum_algorithm(file.read())
    return file_hash.hexdigest()

def download_file(
    url,
    filename=None,
    path=None,
    tempdir=None,
    checksum=None,
    checksum_algorithm=hashlib.sha256):
    '''
    Taken and modified from
    http://stackoverflow.com/a/16518224/555565
    '''
    u = urllib2.urlopen(url)

    if not filename:
        scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
        filename = os.path.basename(path)
        if not filename:
            filename = 'downloaded.file'
    if path:
        filename = os.path.join(path, filename)

    with tempfile.NamedTemporaryFile('wb+', dir=tempdir, delete=False) as f:
        meta = u.info()
        meta_func = meta.getheaders if hasattr(meta, 'getheaders') else meta.get_all
        meta_length = meta_func('Content-Length')
        file_size = None
        if meta_length:
            file_size = int(meta_length[0])
        print('Downloading {0} to {1} ({2} bytes)...'.format(url, filename, file_size))

        file_size_dl = 0
        block_sz = 8192
        while True:
            buffer = u.read(block_sz)
            if not buffer:
                break

            sys.stdout.write('\r')
            file_size_dl += len(buffer)
            f.write(buffer)

            status = '{0:16}'.format(file_size_dl)
            if file_size:
                status += '   [{0:6.2f}%] '.format(file_size_dl * 100 / file_size)
            sys.stdout.write(status)
            sys.stdout.flush()

        # checksum file before moving it into place
        if checksum:
            file_checksum = checksum_file(f, checksum_algorithm)
            if checksum != file_checksum:
                os.remove(f.name)
                print('')
                raise Exception('checksum failed: {1} != {2}'.format(url, checksum, file_checksum))
            else:
                sys.stdout.write('checksum: {0}'.format(checksum))

        # Great!  All went well.  Move the file to its final location.
        shutil.move(f.name, filename)
        print(' - Done')

    return filename

artifacts.json

[
    {
        "name": "screech.mp3",
        "url":  "http://ottomata.org/files/media/audio/southeast_asia/screech.mp3",
        "path": "libs/screech.mp3",
        "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
    },
    {
        "name": "ice_cream.mp3",
        "url":  "http://ottomata.org/files/media/audio/southeast_asia/ice_cream.mp3",
        "path": "libs/ice_cream.mp3",
        "checksum": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
        "checksum_algorithm": "sha1"
    }
]