s3.py

from jumpscale import j
import click, uuid, csv
from multiprocessing import Process, Manager

logger = j.logger.get('s3_deployment')
minio_results_file = 'minio_results_file.csv'

#python3 s3.py --robot local --s3_instances 1 --zt_token ****** --zt_nw_id 35c192ce9b3fecb5 --farmer_name tlre --storage_type hdd --storage_size 5 --minio_login minio_login --minio_password **

@click.command()
@click.option("-r", "--robot", help="0-robot instance to use", required=True)
@click.option("-n", "--s3_instances", help="total number of s3 instances", required=True)
@click.option("-t", "--zt_token", help="zero tier token", required=True)
@click.option("-i", "--zt_nw_id", help="zero tier network id", required=True)
@click.option("-f", "--farmer_name", help="farmer name", required=True)
@click.option("-st", "--storage_type", help="s3 storage type hdd or ssd", required=True)
@click.option("-ss", "--storage_size", help="s3 storage size in G", required=True)
@click.option("-l", "--minio_login", help="minio login", required=True)
@click.option("-p", "--minio_password", help="minio password", required=True)
def main(robot, s3_instances, zt_token, zt_nw_id, farmer_name, storage_type, storage_size, minio_login, minio_password):
    if robot:
        robot = j.clients.zrobot.robots[robot]
    else:
        j.clients.zrobot.get('demo', data={'url': 'http://localhost:6600'})
        robot = j.clients.zrobot.robots['local']

    zerotier_client = str(uuid.uuid4()).replace('-', '')[:10]
    logger.info(" [*] zerotier client : {}".format(zerotier_client))
    zt_data = {
    'token': zt_token
    }
    robot.services.create("github.com/threefoldtech/0-templates/zerotier_client/0.0.1", zerotier_client, data=zt_data)

    s3_data = {
        'farmerIyoOrg': farmer_name,
        'mgmtNic': {'id': zt_nw_id, 'ztClient': zerotier_client},
        'storageType': storage_type,
        'storageSize': int(storage_size),
        'minioLogin': minio_login,
        'minioPassword': minio_password
    }    

    m = Manager()
    results_que = m.Queue()   
    s3_services_queue = m.Queue()
    procs = [] 
    for s3_instance in range(int(s3_instances)):
        pro = Process(target=install_s3, args=(robot, s3_data, results_que, minio_login, minio_password,s3_services_queue, ))
        pro.start()
        procs.append(pro)

    for pro in procs:
        pro.join()
    export_data_to_csv_file(results_que)     

def install_s3(robot, data, que, minio_login, minio_password, s3_services_queue):
    s3_service_name = str(uuid.uuid4()).replace('-', '')[:10]
    logger.info(" [*] s3 service name : {}".format(s3_service_name))
    s3 = robot.services.create('github.com/threefoldtech/0-templates/s3/0.0.1', s3_service_name, data)
    s3.schedule_action('install').wait(die=True)
    
    minio_url = s3.schedule_action('url').wait(die=True)
    minio_access_point = minio_url.result['storage'][7:] #raises an error!

    que.put({'interface':minio_access_point, 'username': minio_login, 'password': minio_password})  
    s3_services_queue.put(s3)

def export_data_to_csv_file(data_que):
    import ipdb; ipdb.set_trace()
    with open(minio_results_file, 'w') as csvfile:
        fieldnames = ['interface', 'username', 'password']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

        writer.writeheader()
        while not data_que.empty():
            writer.writerow(data_que.get())

def minio_info(self):
    return {self.data['minioUrl']: self._nodes[0]['node_id']}

if __name__ == "__main__":
    main()