gistfile1.py

import rethinkdb as r
import sys
import threading
import time

from datetime import datetime
from random import randint

from faker import Faker

f = Faker()
now = str(datetime.now())
conn = r.connect('localhost', 28015, db='corvisacloud')

batch_count = 0
batch_size = 15000
leads = []
threads = []

def batch_insert(leads):
    print("\nBatch populated - Pushing to DB")
    start_time = time.time()
    r.table('lead').insert(leads).run(conn, noreply = True, durability = 'soft')
    print("Insert executed in {0} seconds".format(time.time() - start_time))

for i in xrange(0,1000000):
    rand_int = randint(1000,10000)
    _bool = rand_int % 2 == 0
    first_name = f.first_name()
    last_name = f.last_name()
    name = "{0} {1}".format(first_name,last_name)
    lead = {
        "activation_date": now,
        "annual_revenue": "${0}".format(rand_int),
        "city": f.city(),
        "clean_status": _bool,
        "company": f.company(),
        "company_duns_number": i,
        "connection_received_id": i,
        "connection_sent_id": i,
        "converted_account_id": i,
        "converted_contact_id": i,
        "converted_date": now,
        "converted_opportunity_id": i,
        "country": f.city(),
        "country_code": i,
        "created": now,
        "created_by_id": i,
        "created_date": now,
        "currency_iso_code": i,
        "deactivation_date": now,
        "description": f.lorem(),
        "division": 'null',
        "effective_date": now,
        "email": f.email(),
        "email_bounced_date": now,
        "email_bounced_reason": 'null',
        "expiration_date": now,
        "fax": f.phonenumber(),
        "first_name": first_name,
        "has_opted_out_of_email": _bool,
        "industry": 'null',
        "is_converted": _bool,
        "is_deleted": _bool,
        "is_unread_by_owner": _bool,
        "id":i,
        "jigsaw": i,
        "last_activity_date": now,
        "last_modified_by_id": 'null',
        "last_modified_date": now,
        "last_name": last_name,
        "last_referenced_date": now,
        "last_viewed_date": now,
        "latitude": 'null',
        "lead_source": 'null',
        "longitude": 'null',
        "master_record_id": i,
        "mobile_phone": f.phonenumber(),
        "modified": now,
        "name": name,
        "number_of_employees": 'null',
        "owner_id": '005G0000002CeJkIAK',
        "partner_account_id": 'null',
        "phone": f.phonenumber(),
        "postal_code": f.zip_code(),
        "rating": 'null',
        "record_type_id": '012G0000000ySB7IAM',
        "salesforce_id": i + randint(1,100000),
        "salutation": 'Mr.',
        "state": 'Wisconsin',
        "state_code": f.state(),
        "status": 'Live',
        "street": 'null',
        "system_mod_timestamp": 'null',
        "title": 'null',
        "website": 'null'
    }

    if batch_count >= batch_size:

        t = threading.Thread(target=batch_insert, args=[leads])
        threads.append(t)
        t.start()

        batch_count = 0
        leads = []

    else:
        sys.stdout.write(".")
        sys.stdout.flush()
        leads.append(lead)
        batch_count += 1