gistfile1.txt

import cStringIO
import multiprocessing
import os
import pickle
import struct
import sys
import zlib

class MyProcess(multiprocessing.Process):
    def __init__(self, s):
        self.f = os.fdopen(s)
        super(MyProcess, self).__init__()

    def run(self):
        size = struct.unpack('!Q', self.f.read(8))[0]
        data = cStringIO.StringIO()
        while data.tell() < size:
            read_len = 100000
            if size - data.tell() < read_len:
                read_len = size - data.tell()
            data.write(self.f.read(read_len))
        new_obj = pickle.loads(data.getvalue())

num_entries = int(sys.argv[1])
a = dict()
for i in xrange(num_entries):
    a['a%010d' % i] = 'a'*i
pa = pickle.dumps(a, pickle.HIGHEST_PROTOCOL)

def test():
    pr, pw = os.pipe()
    p = MyProcess(pr)
    p.start()
    #print("sending %d bytes" % len(pa))
    packed_len = struct.pack('!Q', len(pa))
    os.write(pw, packed_len)
    os.write(pw, pa)
    p.join()

import datetime

for i in range(10):
    now = datetime.datetime.now()
    test()
    diff = datetime.datetime.now() - now
    print("run %d: %s" % (i, diff))


[root@jimi v2perf]# python test.py 1000
run 0: 0:00:00.006883
run 1: 0:00:00.005624
run 2: 0:00:00.005201
run 3: 0:00:00.005377
run 4: 0:00:00.004965
run 5: 0:00:00.004765
run 6: 0:00:00.006735
run 7: 0:00:00.005441
run 8: 0:00:00.005372
run 9: 0:00:00.005652
[root@jimi v2perf]# python test.py 5000
run 0: 0:00:00.038560
run 1: 0:00:00.041640
run 2: 0:00:00.041586
run 3: 0:00:00.041806
run 4: 0:00:00.042107
run 5: 0:00:00.040870
run 6: 0:00:00.041143
run 7: 0:00:00.041276
run 8: 0:00:00.042075
run 9: 0:00:00.038851
[root@jimi v2perf]# python test.py 10000
run 0: 0:00:00.115320
run 1: 0:00:00.120042
run 2: 0:00:00.117690
run 3: 0:00:00.116884
run 4: 0:00:00.114376
run 5: 0:00:00.115318
run 6: 0:00:00.116994
run 7: 0:00:00.116928
run 8: 0:00:00.114742
run 9: 0:00:00.123402