Skip to content

Instantly share code, notes, and snippets.

@natekupp
Created September 26, 2015 19:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save natekupp/33bdc7b7324e68afbe8f to your computer and use it in GitHub Desktop.
Save natekupp/33bdc7b7324e68afbe8f to your computer and use it in GitHub Desktop.
impyla+multiprocessing crash example
import multiprocessing
import impala.dbapi
class Impala(obect):
def __init__(self, host, database, port, username, password, lock=None):
self.conn = impala.dbapi.connect(host=host,
port=port,
database=database,
user=username,
password=password,
auth_mechanism='PLAIN',
use_ssl=True)
self.lock = lock or multiprocessing.Lock()
def run_query(self, query):
with self.lock:
cur = self.conn.cursor()
cur.execute(query)
cur.close()
if __name__ == '__main__':
queries = [] # some list of SQL queries
lock = multiprocessing.Lock()
impala = Impala(..., lock=lock)
pool = multiprocessing.Pool(8)
pool.map(impala.run_query, queries)
pool.close()
pool.join()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment