Skip to content

Instantly share code, notes, and snippets.

@kmuthukk
Last active May 25, 2019 00:20
Show Gist options
  • Save kmuthukk/b2d1132081eecfa8289473d3ab375a3c to your computer and use it in GitHub Desktop.
Save kmuthukk/b2d1132081eecfa8289473d3ab375a3c to your computer and use it in GitHub Desktop.
Sample python program to load to a table with a secondary index using YugaByte DB's postgres compatible SQL API
import psycopg2;
from multiprocessing.dummy import Pool as ThreadPool
num_write_threads=2
num_users=10000
def create_table():
conn = psycopg2.connect("host=localhost dbname=postgres user=postgres port=5433")
conn.set_session(autocommit=True)
cur = conn.cursor()
cur.execute("""DROP TABLE IF EXISTS users""");
cur.execute("""CREATE TABLE IF NOT EXISTS users(
id text,
ename text,
age int,
PRIMARY KEY(id))
""")
print("Created users table")
print("====================")
cur.execute("""
CREATE INDEX IF NOT EXISTS name_idx ON users(ename)
""")
print("Created name_idx on table")
def load_data_slave(thread_num):
thread_id = str(thread_num)
conn = psycopg2.connect("host=localhost dbname=postgres user=postgres port=5433")
conn.set_session(autocommit=True)
cur = conn.cursor()
print("Thread-" + thread_id + ": Inserting %d rows..." % (num_users))
# "name" column is indexed, and may have duplicates across threads.
for idx in range(num_users):
cur.execute("""INSERT INTO users (id, ename, age) VALUES (%s, %s, %s)""",
("user-"+thread_id+"-"+str(idx),
"name--"+str(idx),
20 + (idx % 50)))
print("Thread-" + thread_id + ": Inserted %d rows" % (num_users))
def load_data():
pool = ThreadPool(num_write_threads)
results = pool.map(load_data_slave, range(num_write_threads))
# Main
create_table()
load_data()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment