transfer1.py

import MySQLdb
from MySQLdb.cursors import SSCursor
from google.cloud import datastore
from google.cloud.exceptions import TooManyRequests

import json
import time

def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i + n]

# Instantiates a client
datastore_client = datastore.Client()

# The kind for the new entity
kind = 'Part'

i = 0
ii = 0

conn = MySQLdb.connect(host = "xxx" , user = "xxx", passwd = "xxx", db = "xxx", cursorclass=SSCursor)
cursor = conn.cursor()
cursor.execute("SELECT * FROM table;")

while True:
    rows = cursor.fetchmany(250)
    if rows == ():
        break
    updates = []
    for row in rows:
        # The name/ID for the new entity
        name = row[0]
        # The Cloud Datastore key for the new entity
        task_key = datastore_client.key(kind, name)

        # Prepares the new entity
        task = datastore.Entity(key=task_key)
        data = json.loads(row[2])
        for key in data:
            task[key] = data[key]

        updates.append(task)
    i += len(updates)

    done = False
    count = 1
    while not done:
        try:
            datastore_client.put_multi(updates)
            done = True
        except TooManyRequests as e:
            print count, e
            time.sleep(count)
            count *= 2
            count = min(600, count)
    
    
conn.close()