Read from MooseFS, write to QFS

read_mfs_write_qfs.py

"""
- sequence read
- sequence write 
"""

import os
import sys
import time
import errno
import time 

import qfs

def ParseConfig(config):
    host = ''
    port = -1
    for line in open("qfssample.cfg"):
        if line.startswith("#") or len(line.strip()) == 0:
            continue
        s = line.strip()
        if s.split('=')[0].strip() == 'metaServer.name':
            host = s.split('=')[1].strip()
        elif s.split('=')[0].strip() == 'metaServer.port':
            port = int(s.split('=')[1].strip())
    if (host,port) == ('', -1):
        sys.exit('Failed to parse config file')
    return (host,port)


def createDirectories(Client, Url, parentFolder):
	LF   = Url.split('/')
	lenL = len (LF)
	index = 0
	for i in LF:
		if(index  < lenL - 1 and i != None and i != ''):
			parentFolder = parentFolder + i + "/"
			try:
				Client.mkdir(parentFolder)
				print "Create %s" %(parentFolder)
			except IOError, err:
				pass
		index = index + 1

def main():
	if len(sys.argv) < 2:
		sys.exit('Usage: %s config_file' % sys.argv[0])
	client = None
	server = ParseConfig(sys.argv[1])
	listFile = "allPhoto.txt"
	f = open(listFile)
	try:
		client = qfs.client(server)
	except:
		print "Unable to start the QFS client."
		print "Make sure that the meta- and chunkservers are running."
		sys.exit(1)

	parentFolder = "/"
	client.cd(parentFolder)
	for node in client.readdir(parentFolder): print node
	i = 0 
	begin = time.time()
	for url in f.readlines(): 
		photoFile = url.split('/')[-1]
		recursiveDir = url.replace(photoFile, '') 
		createDirectories(client, recursiveDir, parentFolder) 
		client.cd(recursiveDir)
		print photoFile 
		dfsFile = client.create(photoFile.split('.')[0], 1) 
		binaryData = open(url.strip(), 'rb').read() 
		dfsFile.write(binaryData) 
		dfsFile.sync()
		dfsFile.close() 
		i += 1
	print "Read %s from MFS and write QFS %s in %s seconds" %(i, i, (time.time() - begin)/1000 )
	print "Per second: " + i/(time.time() - begin)/1000 
	 
	
if __name__ == '__main__':
    main()