memuse - measure a total unique RAM taken by process tree

memuse.md

memuse measures unique physical total memory taken by a process and its children, ignoring duplicate copy-on-write pages and shared memory.

This is a solution for http://serverfault.com/questions/676335/how-measure-memory-without-copy-on-write-pages

It's a quick and dirty utility, but feel free to fork & improve.

Example:

~ » sudo ./memuse.py 15897                                                                                                                         eugene@eugene-thinkpad
PID                 Commandline                          Frames (+unique)           VMEM
 - 15897            (/usr/bin/zsh                  ):      1776  +1776           7104 KB
  -  9783           (sudo make rundev              ):       608  +408            2432 KB
   -  9784          (make rundev                   ):       261  +98             1044 KB
    -  9785         (/bin/sh -c cd ajenti-panel && ):       166  +48              664 KB
     -  9786        (python ./ajenti-panel -v --aut):      9279  +8977          37116 KB
      -  9795       (./ajenti-panel worker [restric):      7637  +1334          30548 KB
      -  9834       (./ajenti-panel worker [session):      8972  +2639          35888 KB
----------------------------------------------------------------------------------------
TOTAL:                                                    15280                 61120 KB

See also:

• http://stackoverflow.com/questions/1401359/understanding-linux-proc-id-maps
• https://www.kernel.org/doc/Documentation/vm/pagemap.txt

memuse.py

#!/usr/bin/env python
import psutil
import struct
import os
import sys

PAGESIZE = 4096


class Map(object):
    pass


def get_pages(pid):
    maps = []
    for l in open('/proc/%i/maps' % pid):
        tokens = l.split()
        m = Map()
        m.mfrom = int(tokens[0].split('-')[0], 16)
        m.mto = int(tokens[0].split('-')[1], 16)
        maps.append(m)

    pfns = set()

    f = open('/proc/%i/pagemap' % pid, 'rb')
    for map in maps:
        for vpage in range(map.mfrom / PAGESIZE, map.mto / PAGESIZE + 1):
            f.seek(8 * vpage)
            bytes = f.read(8)
            if bytes:
                page = struct.unpack('<Q', bytes)[0]
                pfn = page & 0x7FFFFFFFFFFFFF

                swapped = bool(page & (2 ** 62))
                present = bool(page & (2 ** 63))

                pfns.add(pfn)

    return pfns

try:
    pid = int(sys.argv[1])
except:
    print 'Usage: memuse.py <pid>'
    sys.exit(1)


whole = set()

def scan(pid, level=0):
    p = psutil.Process(pid)
    pfns = get_pages(p.pid)
    old_frames = len(whole)
    for pfn in pfns:
        whole.add(pfn)
    new_frames = len(whole)
    print '%s - %5i %s (%-030s):  %8i  +%-8i %10i KB' % (
        ' ' * level,
        p.pid,
        ' ' * (10 - level),
        ' '.join(p.cmdline())[:30],
        len(pfns),
        (new_frames - old_frames),
        len(pfns) * PAGESIZE / 1024
    )
    for c in p.children():
        scan(c.pid, level=level+1)

print 'PID                 Commandline                          Frames (+unique)           VMEM'
scan(pid)
print '----------------------------------------------------------------------------------------'
print 'TOTAL:                                                 %8i            %10i KB' % (
    len(whole),
    len(whole) * PAGESIZE / 1024
)