Last active
May 25, 2019 00:20
-
-
Save kmuthukk/b2d1132081eecfa8289473d3ab375a3c to your computer and use it in GitHub Desktop.
Sample python program to load to a table with a secondary index using YugaByte DB's postgres compatible SQL API
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import psycopg2; | |
from multiprocessing.dummy import Pool as ThreadPool | |
num_write_threads=2 | |
num_users=10000 | |
def create_table(): | |
conn = psycopg2.connect("host=localhost dbname=postgres user=postgres port=5433") | |
conn.set_session(autocommit=True) | |
cur = conn.cursor() | |
cur.execute("""DROP TABLE IF EXISTS users"""); | |
cur.execute("""CREATE TABLE IF NOT EXISTS users( | |
id text, | |
ename text, | |
age int, | |
PRIMARY KEY(id)) | |
""") | |
print("Created users table") | |
print("====================") | |
cur.execute(""" | |
CREATE INDEX IF NOT EXISTS name_idx ON users(ename) | |
""") | |
print("Created name_idx on table") | |
def load_data_slave(thread_num): | |
thread_id = str(thread_num) | |
conn = psycopg2.connect("host=localhost dbname=postgres user=postgres port=5433") | |
conn.set_session(autocommit=True) | |
cur = conn.cursor() | |
print("Thread-" + thread_id + ": Inserting %d rows..." % (num_users)) | |
# "name" column is indexed, and may have duplicates across threads. | |
for idx in range(num_users): | |
cur.execute("""INSERT INTO users (id, ename, age) VALUES (%s, %s, %s)""", | |
("user-"+thread_id+"-"+str(idx), | |
"name--"+str(idx), | |
20 + (idx % 50))) | |
print("Thread-" + thread_id + ": Inserted %d rows" % (num_users)) | |
def load_data(): | |
pool = ThreadPool(num_write_threads) | |
results = pool.map(load_data_slave, range(num_write_threads)) | |
# Main | |
create_table() | |
load_data() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment