pandas leak on pypy

pandasleak.py

import sys
import pandas as pd
import numpy as np
from plotmem import Plotmem
import gc

def leak1():
    for i in Plotmem(100):
        df = pd.DataFrame(
            {'A': ['foo', 'bar'],
             'D': np.random.randn(2)})
        x = df.groupby(['A'])
        x.describe()
        if i % 10 == 0:
            sys.stdout.write('.')
            sys.stdout.flush()
            gc.collect()

def leak2():
    # this is the same as leak1, but instead of calling "describe" we manually
    # call something which is inside it, with a bit of simplificaition
    for i in Plotmem(100):
        df = pd.DataFrame(
            {'A': ['foo', 'bar'],
             'D': np.random.randn(2)})
        x = df.groupby(['A'])
        x._set_group_selection()
        for j in range(10): # calling apply 10 times makes the leak faster I
            # *think* that the leak is generated somewhere in
            # pandas.core.groupby.FrameSplitter.fast_apply. Try to comment out
            # lib.apply_frame_axis0(sdata, f, names, starts, ends) and the
            # leak seems to vanish
            x.apply(lambda y: y)
        if i % 10 == 0:
            sys.stdout.write('.')
            sys.stdout.flush()
            gc.collect()


leak2()

plotmem.py

import re
import os

RE_VMRSS = re.compile("VmRSS:\s*(\d+)")
def get_vmrss():
    filename = '/proc/%d/status' % os.getpid()
    with open(filename) as f:
        s = f.read()
        match = RE_VMRSS.search(s)
        if not match:     # VmRSS is missing if the process just finished
            return None
        rss = int(match.group(1))
        return rss


def plot_terminal(numbers, title):
    import subprocess
    def color(string):
        green = '32;01'
        return '\x1b[%sm%s\x1b[00m' % (green, string)
    if title:
        title = "title '%s'" % title
    else:
        title = 'notitle'
    p = subprocess.Popen(['gnuplot', '-e', 'set term dumb 140, 25; plot "-" %s' % title],
                         stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    stdin = '\n'.join(map(str, numbers))
    stdout, stderr = p.communicate(stdin)
    stdout = stdout.replace('A', color('*'))
    print stdout

class Plotmem(object):

    def __init__(self, N=100):
        self.N = N
        self._mem = []

    def __iter__(self):
        self._mem.append(get_vmrss())
        for i in xrange(self.N):
            yield i
            self._mem.append(get_vmrss())
        plot_terminal(self._mem, '')