impalaya-sample

impala-sample

#!/usr/bin/python
# -*- coding:utf-8 -*-

from impala.dbapi import connect
import gevent
from impala.error import *
import time


def fetch(sql,func):
    (conn,cursor)=dbconnect()
    f0=time.time()
    try:
        cursor.execute('USE nicodata')
        print cursor.description # prints the result set's schema
        print 'precomment  execute'
        gevent.sleep(0)
        r=cursor.execute(sql)
        print 'execute {0} result {1}'.format(r,sql)
        gevent.sleep(0)
        print 'post comment execute'
        print cursor.description # prints the result set's schemaresults = cursor.fetchall()
        results = cursor.fetchall()
        for r in results:
            func(r)
        #print r
    except RPCError,err:
        print err
    finally:
        disconnect(conn,cursor)    
        print 'post comment fetchall'
    out=time.time()-f0
    print "out {0}.sec".format(out)

def fetch_test(tblname):
    sql='SELECT count(smid) FROM nicodata.{0} WHERE comment_string like "%88888%"'.format(tblname)
    def pnt(x):
        print x[0]
    fetch(sql,pnt)

def fetch_video_test():
    sql='SELECT * FROM nicodata.videoinfo_sampling100 LIMIT 3'
    def pnt(x):
        print x[0],x[2]
    fetch(sql,pnt)

def dbconnect():    
    conn = connect(host='ec2-54-249-180-58.ap-northeast-1.compute.amazonaws.com', port=21050)
    cursor = conn.cursor()
    print cursor
    return conn,cursor

def disconnect(conn,cursor):
    cursor.close()
    conn.close()

def main():
    f0=time.time()
    tblname1='comment_data_sampling100'
    tblname2='comment_data_sampling10'
    tblname3='comment_data'
 
    gevent.joinall([
        gevent.spawn(fetch_test,tblname1),
        gevent.spawn(fetch_test,tblname2),
        #gevent.spawn(fetch_test,tblname3),
        gevent.spawn(fetch_video_test),
    ])
    out=time.time()-f0
    print 'all spent time={0}'.format(out)

if __name__=='__main__':main()