A script to upload a folder recursively to Midas.

pydas-upload.py

#!/usr/bin/env python

description = """
Upload binary data to Midas.
"""

import argparse
import hashlib
import os
import sys

import pydas


def connect_to_midas(email=None, api_key=None):
    midas_url = 'http://midas3.kitware.com/midas/'
    if not api_key:
        print('Please enter your login information for ' + midas_url)
        pydas.login(url=midas_url, email=email)
    else:
        pydas.login(url=midas_url, email=email, api_key=api_key)
    session = pydas.session
    communicator = session.communicator
    return session, communicator


def upload_to_midas(input_file,
                    root,
                    folder_prefix,
                    community_name,
                    session,
                    communicator):
    input_path = os.path.join(root, input_file)
    input_fp = open(input_path, 'rb')
    # get the MD5 checksum
    print('Uploading: ' + input_path)
    md5 = hashlib.md5()
    for chunk in iter(lambda: input_fp.read(128 * md5.block_size), b''):
        md5.update(chunk)
    md5hash = md5.hexdigest()
    print('Checksum: ' + md5hash)

    # upload to Midas
    def get_child_folder(parent, child_name):
        children = communicator.folder_children(session.token,
                                                parent['folder_id'])
        for folder in children['folders']:
            if folder['name'] == child_name:
                return folder
        return None

    community = communicator.get_community_by_name(community_name)
    # The first folder should always be "Public" or "Private"
    dest_folders = os.path.join(folder_prefix, root)
    folders = os.path.normpath(dest_folders).split(os.path.sep)
    current_folder = get_child_folder(community, folders[0])

    if len(folders) > 1:
        for folder in folders[1:]:
            child_folder = get_child_folder(current_folder, folder)
            if child_folder is None:
                print('Creating folder: ' + folder)
                current_folder = \
                    communicator.create_folder(session.token,
                                               folder,
                                               current_folder['folder_id'])
            else:
                current_folder = child_folder

    # get the existing or create a new item to hold the file
    item_name = input_file
    item_id = None
    folder_id = current_folder['folder_id']
    current_folder_children = communicator.folder_children(session.token,
                                                           folder_id)
    if 'items' in current_folder_children:
        for item in current_folder_children['items']:
            if item['name'] == item_name:
                item_id = item['item_id']
                break

    if item_id is None:
        new_item = communicator.create_item(session.token, item_name,
                                            current_folder['folder_id'])
        item_id = new_item['item_id']

    upload_token = communicator.generate_upload_token(session.token,
                                                      item_id,
                                                      item_name,
                                                      md5hash)
    if upload_token != "":
        communicator.perform_upload(upload_token,
                                    item_name,
                                    item_id=item_id,
                                    revision='head',
                                    filepath=input_path)

    input_fp.close()


def run(input_paths, folder_prefix, community,
        email=None, api_key=None):
    session, communicator = connect_to_midas(email, api_key)

    for input_path in input_paths:
        if os.path.isfile(input_path):
            upload_to_midas(input_path, '.', folder_prefix, community,
                            session, communicator)
        elif os.path.isdir(input_path):
            for root, dirs, files in os.walk(input_path):
                for ff in files:
                    upload_to_midas(ff, root, folder_prefix, community,
                                    session, communicator)
        else:
            print('Unknown path: ' + input_path)
            sys.exit(1)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('--api-key-file', '-k', type=argparse.FileType('r'),
                        help="A file that contains your Midas user's API key.")
    parser.add_argument('--email', '-e',
                        help="Email address associated with Midas account.")
    parser.add_argument('--prefix', '-p', default='Public',
                        help="Folder to place the inputs on Midas")
    parser.add_argument('--community', '-c', default='TubeTK',
                        help="Midas community.")
    parser.add_argument('input_paths', nargs='+')
    args = parser.parse_args()

    if args.api_key_file:
        api_key = args.api_key_file.readline()
        api_key = api_key.strip()
    else:
        api_key = None

    run(args.input_paths, args.prefix, args.community,
        email=args.email, api_key=api_key)