Merge two or more databases created by LogReqRes plugin for proxenet

proxenet-logreqres-merge.py

#!/usr/bin/env python2.7
#
# Merge two or more databases created by LogReqRes plugin for proxenet
#

import sys, sqlite3


def init_merge_database(cur):
    try:
        cur.execute("CREATE TABLE requests  (id INTEGER, request BLOB, uri TEXT, timestamp INTEGER)")
        cur.execute("CREATE TABLE responses (id INTEGER, response BLOB,  uri TEXT, timestamp INTEGER)")
        return True
    except:
        return False


def merge_database( merge_cur, cur_cur, offset ):
    n1 = n2 = 0

    for row in cur_cur.execute("SELECT id, request, uri, timestamp FROM requests"):
        try :
            rid, req, uri, ts = row
            merge_cur.execute('INSERT INTO requests VALUES (?,?,?,?)', (offset+rid, req, uri, ts))
            n1 = offset+rid
        except Exception as e:
            print e
            return -1

    for row in cur_cur.execute("SELECT id, response, uri, timestamp FROM responses"):
        try :
            rid, res, uri, ts = row
            merge_cur.execute('INSERT INTO responses VALUES (?,?,?,?)', (offset+rid, res, uri, ts))
            n2 = offset+rid
        except Exception as e:
            print e
            return -1

    return max(n1, n2)


if __name__ == "__main__":

    files = sys.argv[1:]

    if len(files)==0:
        print "Missing databases"
        exit(1)

    if len(files)==1:
        print "Need more than 1 database to merge"
        exit(0)

    merge_db_name = 'proxenet-merge.db'
    merge_db_con = sqlite3.connect(merge_db_name)
    merge_db_con.text_factory = str
    merge_db_cur = merge_db_con.cursor()

    if not init_merge_database(merge_db_cur):
        print "Failed to init merge database"
        merge_db_con.close()
        exit(1)

    merge_db_con.commit()

    offset = 0

    for f in files:
        try:
            cur_db_con = sqlite3.connect(f)
            cur_db_con.text_factory = str
            cur_db_cur = cur_db_con.cursor()
            n = merge_database( merge_db_cur, cur_db_cur, offset )

            if n < 0:
                print "Error for '%s'" % f
                continue

            merge_db_con.commit()
            offset += n
        except Exception as e:
            print "Error %s:" % e

        finally:
            if cur_db_con:
                cur_db_con.close()

    merge_db_con.close()