ZODB Tranalyzer (tranalyzer.py)

tranalyzer.py

#!/usr/bin/env python
#
#	$Id: tranalyzer.py,v 1.5 1999/10/22 14:23:55 tsarna Exp tsarna $
#
# Copyright (c) 1999 Tyler C. Sarna
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. The name of the author may not be used to endorse or promote products
#    derived from this software without specific prior written permission
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import sys, struct, time, getopt, string, cStringIO, cPickle

t32 = 1L << 32

def oid2x64(v):
    h, v = struct.unpack(">ii", v)
    if v < 0: v=t32-v
    if h:
        if h < 0: h=t32-h
        v=h*t32+v
    v = hex(v)[2:]
    if v[-1] == 'L':
        v = v[:-1]
    return v

def refs(p):
    """Get list of references from a pickle"""
    s = cStringIO.StringIO(p)
    u = cPickle.Unpickler(s)
    u.persistent_load = rl = []
    try:
        u.noload()
        u.noload()
    except:
        pass
    l = []
    for r in rl:
        if type(r) == type(()):
            r = r[0]
        l.append(oid2x64(r))
    return l

class FSFile:
    """File wrapper with some parsing additions and that keeps track of
    the sizes of subregions, and which can wait for more data to be
    written to the file"""
    
    def __init__(self, f, wait=None):
        self.f = f
        self.lefts = []
        self.left = None
        self.wait = wait

    def read(self, n):
        s = self.f.read(n)
        if (len(s) != n) and not self.wait:
            raise IOError
        while (len(s) != n) and self.wait:
            time.sleep(1)
            s = s + self.f.read(n - len(s))
        return s
        
    def checkleft(self, n):
        if self.left is not None:
            if n > self.left:
                print self.left, n
                raise 'Not enough data left, %d < %d' % (self.left, n)
            self.left = self.left - n
                            
    def pushLeft(self, n):
        self.lefts.append(self.left)
        self.left = n

    def popLeft(self):
        oldleft, self.left = self.left, self.lefts.pop()
        return oldleft

    def addLeft(self, n):
        self.left = self.left + n
        
    def tell(self):
        return self.f.tell()
        
    def c(self, n):
        self.checkleft(n)
        return self.read(n)
        
    def u16(self):
        self.checkleft(2)
        v = self.read(2)
        v = struct.unpack(">H", v)[0]
        return v
    
    def u32(self):
        self.checkleft(4)
        v = self.read(4)
        v = struct.unpack(">i", v)[0]
        return v
    
    def u64(self):
        self.checkleft(8)
        v = self.read(8)
        h, v = struct.unpack(">ii", v)
        if v < 0: v=t32-v
        if h:
            if h < 0: h=t32-h
            v=h*t32+v
        return v

    def x64(self):
        v = hex(self.u64())[2:]
        if v[-1] == 'L':
            v = v[:-1]
        return v


class Transaction:
    """Reads information about a transaction from a FSFile"""

    def __init__(self, f, off, refs=0):
        self.off = off
        try:
            self.tid = f.x64()
        except IOError:
            # not truncated, just EOF
            raise EOFError
        self.tlen = f.u64() - 8
        f.pushLeft(self.tlen - 8)

        self.status = f.c(1)
        self.xstatus = ''
        if self.status != ' ':
            self.xstatus = "(status '%c') " % self.status

        userlen, desclen, self.xalen = f.u16(), f.u16(), f.u16()
        
        self.user = f.c(userlen)
	if not self.user:
		self.user = "[Zope]"

        self.desc = f.c(desclen)
        if self.xalen:
            self.xa = f.c(self.xalen)
            self.xastr = " xalen %d" % self.xalen
        else:
            self.xastr = ""

        self.obs = []
        while f.left:
            self.obs.append(Record(f, refs))

        self.nobs = len(self.obs)
        
        f.popLeft()

        tlen2 = f.u64() - 8
        
        if tlen2 != self.tlen:
            raise ValueError, 'transaction lengths mismatch'

    def __getitem__(self, key):
        return getattr(self, key)
        
    def __str__(self):
        s = """TID: %(tid)s @ %(off)d obs %(nobs)d len %(tlen)d%(xastr)s %(xstatus)sBy %(user)s
"%(desc)s"
""" % self

        for o in self.obs:
            s = s + str(o)

	return s
           

class Record:
    """Read a object record from a FSFile"""
    
    def __init__(self, f, printrefs=0):
        self.oid = f.x64()
        self.serial = f.x64()
        self.prevrec = f.u64()
        self.transtart = f.u64()
        self.refs = []
        self.printrefs = printrefs
        vl = self.verslen = f.u16()
        dl = self.datalen = f.u64()
        
        if vl:
            self.nvpos = f.u64()
            self.nvprevrec = f.u64()
            self.vstring = f.c(vl - 16)
        
        if dl:
            data = f.c(dl - 8)
            if printrefs:
                self.refs = refs(data)
            self.drpos = f.u64()

    def __getitem__(self, key):
        return getattr(self, key)
        
    def __str__(self):
        s = "\tOID: %(oid)s len %(datalen)d\n" % self
        if self.verslen:
            s = s + """\t\t(version "%(vstring)s", vlen %(verslen)d)\n"""
        rl = self.refs
        if self.printrefs:
            while len(rl):
                tr = rl[:4]
                rl = rl[4:]
                s = s + "\t\t" + string.join(tr, ', ') + '\n'
        return s

def sniff_trans(f, o):
    """Does the data at offset o smell like a transaction?"""
    x = f.tell()
    try:
        try:
            f = FSFile(f)
            f.f.seek(o)
            tid = f.u64()
            len = f.u64()
            status = f.c(1)
            if status not in (' ', 'p', 'c', 'u'):
                raise ValueError, "invalid status byte"
                
            if len > 131072:
                raise ValueError, "probably not a valid length"

            f.f.seek(o + len)
            len2 = f.u64()

            if len != len2:
                raise ValueError, "length mismatch"
                
            f.f.seek(o + 17)
            l = f.u16() + f.u16() + f.u16()
            if l > len:
                raise ValueError, "parts bigger than whole"
                
            f.f.seek(o + 17 + 6 + 8 + l)
            ser = f.u64()
            if ser != tid:
                raise ValueError, "object serial should match tid"
            
            f.u64()
            if long(o) != f.u64():
                raise ValueError, "backpointer to transaction is wrong"
            
            ret = 1
        except ValueError:
            ret = 0
    finally:
        f.f.seek(x)

    return ret


def usage():
    sys.stderr.write("""usage: tranalyzer [-f] [-r] [-s offset] file
\t-f\tcontinuous mode (like tail -f)
\t-r\tprint OIDs of referenced objects
\t-s\tseek to first transaction at or after offset
""")
    sys.exit(1)


def main(args):
    optlist, args = getopt.getopt(args[1:], "frs:")
    if len(args) != 1:
        usage()

    if args[0] == '-':
        f = sys.stdin
    else:
        f = open(args[0], 'rb')
    
    seek = cont = refs = 0
    for oname, oarg in optlist:
        if oname == '-f':
            cont = 1
        elif oname == '-r':
            refs = 1
        elif oname == '-s':
            seek = long(oarg)
        else:
            usage()

    f = FSFile(f, cont)
    magic = f.c(4)
    if magic != 'FS21':
        print "error: Not a .fs file (at least, not a kind I know about)"
        sys.exit(1)

    if seek:
        while not sniff_trans(f.f, seek):
            seek = seek + 1
        f.f.seek(seek)

    t = Transaction(f, refs)
    while t:
        print t
        off = f.tell()
        try:
            t = Transaction(f, off, refs)
        except IOError:
            print "****** TRUNCATED TRANSACTION at %d:" % off
            t = None
        except EOFError:
            t = None

if __name__ == "__main__":
    main(sys.argv)