Moves data from a SQL table to MongoDB

transfer.py

from twisted.enterprise import adbapi
from twisted.internet import reactor, defer
import txmongo, sys, random

bottom = int(sys.argv[1]) if len(sys.argv) > 1 else 0
mongo = txmongo.lazyMongoConnectionPool().DATABASE.COLLECTION
sql = adbapi.ConnectionPool(LIBRARY, host=HOST, port=PORT, db=DATABASE, user=USER, passwd=PASS, cp_reconnect=True)

@defer.inlineCallbacks
def main():
    global bottom, mongo, sql
    result = yield sql.runQuery("SELECT MAX(id) FROM parts")
    if not result:
        print "Could not find max id"
        return
    top = result[0][0]
    for start in range(bottom, top+1, 100):
        rows = yield sql.runQuery("SELECT name, seed, answer, added FROM parts ORDER BY id ASC LIMIT %s,100", start)
        documents = []
        for row in rows:
            try:
                row[1].decode("utf8")
                row[2].decode("utf8")
            except:
                continue # If it isn't valid utf-8, who cares?
            documents.append({
                "name": row[0],
                "seed": row[1],
                "answer": row[2],
                "added": row[3],
                "random": random.random()
            })
        yield mongo.insert(documents, safe=True)
        print "Completed {:d} of {:d} rows".format(start, top)

main()
reactor.run()